@frontera-sdk/cli 1.53.23 → 1.53.25

package/src/local/local-knowledge-base-files.ts

@@ -0,0 +1,195 @@
+/**
+ * The files that describe a project's knowledge bases (the Knowledge app's
+ * bases, not Agent knowledge in `knowledge/<name>/`): their TypeScript shapes
+ * and the JSON Schemas `frontera local schema` prints for the two this CLI
+ * owns. A source file's format is the service's (`knowledge-base-source`,
+ * served from the Knowledge source spec), so only its shape is mirrored here.
+ *
+ * ```
+ * knowledge-base/
+ *   settings.json                 KnowledgeBaseSettingsFile
+ *   <name>/knowledge-base.json    KnowledgeBaseFile
+ *   <name>/sources/<source>.json  KnowledgeBaseSourceFile
+ *   <name>/documents/**           uploaded files
+ * agents/<slug>/agent.json        knowledgeBases: KnowledgeBaseRef[]
+ * ```
+ *
+ * People never appear in these files: stewards and document owners stay on
+ * the platform.
+ */
+
+export const KNOWLEDGE_BASE_DIR = 'knowledge-base'
+export const KNOWLEDGE_BASE_FILE = 'knowledge-base.json'
+export const KNOWLEDGE_BASE_SOURCES_DIR = 'sources'
+export const KNOWLEDGE_BASE_SETTINGS_FILE = 'settings.json'
+/** `knowledge-base/<name>/documents/**`: the files uploaded into the base, when the project holds them. */
+export const KNOWLEDGE_BASE_DOCUMENTS_DIR = 'documents'
+
+/** A base's folder name: what `knowledge-base/<name>` names. */
+export const KNOWLEDGE_BASE_NAME_PATTERN = '^[A-Za-z0-9][A-Za-z0-9._-]*$'
+/** `knowledge-base/<name>`: how agent.json names a base, by its folder. */
+export const KNOWLEDGE_BASE_REF_PATTERN = `^${KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_NAME_PATTERN.slice(1)}`
+export type KnowledgeBaseRef = `knowledge-base/${string}`
+
+/** A workspace, by its slug. */
+type WorkspaceSlug = string
+
+export type KnowledgeBaseSharing = { allWorkspaces: true } | { workspaces: WorkspaceSlug[] }
+
+/** `knowledge-base/<name>/knowledge-base.json`. A workspace base is never shared. */
+export type KnowledgeBaseFile = {
+  /** 1 to 200 characters; the folder name when absent. */
+  name?: string
+  description?: string
+} & ({ scope: 'workspace' } | { scope: 'organization'; sharing: KnowledgeBaseSharing })
+
+export type KnowledgeSourceAuth =
+  | { kind: 'none' }
+  | { kind: 'bearer' }
+  | { kind: 'header'; name: string }
+  | { kind: 'basic'; username: string }
+  | { kind: 'oauth_client_credentials'; tokenUrl: string; clientId: string; scope?: string }
+
+type KnowledgeSourceSpec = {
+  kind: 'api_pull'
+  name: string
+  schedule: 'manual' | 'hourly' | 'every_6_hours' | 'daily'
+  keepRemovedDays: 1 | 7 | 30
+  /** Where the list is, paging, changes and fields: checked by the service's schema, carried as it is. */
+  config: Record<string, unknown>
+}
+
+/**
+ * `knowledge-base/<name>/sources/<source>.json`: the service's source spec,
+ * plus the name of the secret (in secrets.json) that holds its credential,
+ * present exactly when the source signs in.
+ */
+export type KnowledgeBaseSourceFile = KnowledgeSourceSpec &
+  ({ auth: { kind: 'none' }; credentialSecret?: never } | { auth: Exclude<KnowledgeSourceAuth, { kind: 'none' }>; credentialSecret: string })
+
+export type KnowledgeSourcePolicy =
+  | { apiSources: 'off' | 'organization' | 'all_workspaces' }
+  | { apiSources: 'chosen_workspaces'; workspaces: WorkspaceSlug[] }
+
+/** `knowledge-base/settings.json`: the organization's Knowledge settings. A key left out leaves that setting alone. */
+export type KnowledgeBaseSettingsFile = {
+  assistant?: {
+    enabled: boolean
+    model: { provider: string; modelId: string } | null
+    name: string | null
+    instructions: string | null
+  }
+  searchModel?: { provider: string; model: string } | null
+  sources?: KnowledgeSourcePolicy
+}
+
+const DRAFT = 'https://json-schema.org/draft/2020-12/schema'
+type JsonSchema = Record<string, unknown>
+
+const WORKSPACES = {
+  type: 'array',
+  minItems: 1,
+  items: { type: 'string', minLength: 1, description: 'A workspace’s slug.' },
+}
+
+const BASE_FIELDS = {
+  name: { type: 'string', minLength: 1, maxLength: 200, description: 'The base’s name as people see it; the folder name when absent.' },
+  description: { type: 'string', description: 'What the base holds, in a sentence.' },
+}
+
+export const KNOWLEDGE_BASE_SCHEMA: JsonSchema = {
+  $schema: DRAFT,
+  title: 'Knowledge base (knowledge-base/<name>/knowledge-base.json)',
+  description:
+    'A knowledge base in the Knowledge app: its name, what it holds, and who may search it. Its API sources are files in sources/ beside this one. An Agent searches it once its agent.json lists `knowledge-base/<name>` in knowledgeBases. A `workspace` base belongs to the project’s workspace and is never shared; an `organization` base belongs to the organization and is shared with every workspace or the ones listed.',
+  type: 'object',
+  oneOf: [
+    {
+      title: 'workspace',
+      type: 'object',
+      properties: { ...BASE_FIELDS, scope: { const: 'workspace', description: 'The project’s workspace owns the base. It cannot be shared.' } },
+      required: ['scope'],
+      additionalProperties: false,
+    },
+    {
+      title: 'organization',
+      type: 'object',
+      properties: {
+        ...BASE_FIELDS,
+        scope: { const: 'organization', description: 'The organization owns the base. Applying it needs a Knowledge steward.' },
+        sharing: {
+          description: 'The workspaces that may search it: `{ "allWorkspaces": true }`, or `{ "workspaces": ["<slug>", ...] }`.',
+          oneOf: [
+            { type: 'object', properties: { allWorkspaces: { const: true } }, required: ['allWorkspaces'], additionalProperties: false },
+            { type: 'object', properties: { workspaces: WORKSPACES }, required: ['workspaces'], additionalProperties: false },
+          ],
+        },
+      },
+      required: ['scope', 'sharing'],
+      additionalProperties: false,
+    },
+  ],
+}
+
+const NULLABLE_TEXT = (maxLength: number, description: string) => ({ anyOf: [{ type: 'string', maxLength }, { type: 'null' }], description })
+
+export const KNOWLEDGE_BASE_SETTINGS_SCHEMA: JsonSchema = {
+  $schema: DRAFT,
+  title: 'Knowledge settings (knowledge-base/settings.json)',
+  description:
+    'The organization’s Knowledge settings: the assistant that answers questions over the bases, the model that makes bases searchable by meaning, and which bases may have API sources. A key left out leaves that setting as the platform has it. Changing them needs permission to update the organization; the assistant also needs a Knowledge steward.',
+  type: 'object',
+  properties: {
+    assistant: {
+      type: 'object',
+      description: 'The Knowledge assistant. Give all four keys.',
+      properties: {
+        enabled: { type: 'boolean', description: 'Whether people can ask the assistant.' },
+        model: {
+          description: 'The model it answers with, or null for the platform’s choice. A model the local runtime lacks is swapped for its default there.',
+          anyOf: [
+            {
+              type: 'object',
+              properties: { provider: { type: 'string', minLength: 1, maxLength: 200 }, modelId: { type: 'string', minLength: 1, maxLength: 300 } },
+              required: ['provider', 'modelId'],
+              additionalProperties: false,
+            },
+            { type: 'null' },
+          ],
+        },
+        name: NULLABLE_TEXT(60, 'The name people see, or null for the default.'),
+        instructions: NULLABLE_TEXT(4000, 'Extra instructions for its answers, or null for none.'),
+      },
+      required: ['enabled', 'model', 'name', 'instructions'],
+      additionalProperties: false,
+    },
+    searchModel: {
+      description: 'The embedding model that makes bases searchable by meaning, or null for the platform’s default. Changing it rebuilds every base’s index.',
+      anyOf: [
+        {
+          type: 'object',
+          properties: { provider: { type: 'string', minLength: 1, maxLength: 100 }, model: { type: 'string', minLength: 1, maxLength: 300 } },
+          required: ['provider', 'model'],
+          additionalProperties: false,
+        },
+        { type: 'null' },
+      ],
+    },
+    sources: {
+      description: 'Which bases may have API sources: none (`off`), the organization’s (`organization`), those and every workspace’s (`all_workspaces`), or those and the listed workspaces’ (`chosen_workspaces` with `workspaces`).',
+      oneOf: [
+        { type: 'object', properties: { apiSources: { enum: ['off', 'organization', 'all_workspaces'] } }, required: ['apiSources'], additionalProperties: false },
+        { type: 'object', properties: { apiSources: { const: 'chosen_workspaces' }, workspaces: WORKSPACES }, required: ['apiSources', 'workspaces'], additionalProperties: false },
+      ],
+    },
+  },
+  additionalProperties: false,
+}
+
+/** agent.json `knowledgeBases`, added to the service's Agent schema by `localizeServiceSchema`. */
+export const KNOWLEDGE_BASES_FIELD: JsonSchema = {
+  type: 'array',
+  description:
+    'The knowledge bases the Agent searches, by folder: `["knowledge-base/<name>", ...]`, each a base in knowledge-base/<name>/knowledge-base.json. When the key is present it is the Agent’s whole list, and an empty list drops every base; leave the key out to keep the bases the Agent has. They are set when the project is applied, live at once, outside the Agent’s draft. Not Agent knowledge: that is `knowledgeAttachments`.',
+  items: { type: 'string', pattern: KNOWLEDGE_BASE_REF_PATTERN, 'x-frontera-reference': [KNOWLEDGE_BASE_DIR] },
+}

package/src/local/local-knowledge-base-pull.ts

@@ -0,0 +1,458 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+import { writeFileAtomic } from '../fs-atomic'
+import {
+  KNOWLEDGE_BASE_DIR,
+  KNOWLEDGE_BASE_FILE,
+  KNOWLEDGE_BASE_NAME_PATTERN,
+  KNOWLEDGE_BASE_SETTINGS_FILE,
+  KNOWLEDGE_BASE_SOURCES_DIR,
+  type KnowledgeBaseFile,
+  type KnowledgeBaseRef,
+  type KnowledgeBaseSettingsFile,
+  type KnowledgeBaseSharing,
+  type KnowledgeSourcePolicy,
+} from './local-knowledge-base-files'
+import { readPulledDocuments, recordPulledDocuments, sha256Of, type DocumentsPulled, type PulledDocument } from './local-knowledge-documents'
+import type { SyncClient } from './local-sync-client'
+import { readLink } from './local-sync-link'
+import { readLock, type RemoteItem } from './local-sync-plan'
+import type { PersonClients, SyncSource } from './local-sync-remote'
+
+/**
+ * Reading the Knowledge app's knowledge bases as `knowledge-base/` holds them
+ * (not Agent knowledge, `knowledge/`). Knowledge reads and writes as a person,
+ * never with a key, so the read is the device session's: as the person stands
+ * in the linked workspace for what that workspace reads, and at the
+ * organization for what only a Knowledge steward or an administrator sees.
+ *
+ *   knowledge-base/<folder>/knowledge-base.json   a base: its name, scope and sharing
+ *   knowledge-base/<folder>/sources/<name>.json   an API source, its credential a secret's name
+ *   knowledge-base/settings.json                  the organization's Knowledge settings
+ *
+ * Documents are not part of an item: they come only when asked for, into
+ * `knowledge-base/<folder>/documents/`, kept out of git.
+ */
+
+const APP = '/v1/knowledge-app'
+
+/** The sync capability of a platform that takes knowledge bases from a project; an older one runs the app but not this. */
+export const KNOWLEDGE_BASES_FEATURE = 'knowledge-bases'
+
+export const KNOWLEDGE_SIGN_IN = 'This workspace’s knowledge bases need a sign-in: run `frontera auth login --browser`, then fetch again'
+const SETTINGS_UNREAD = `Only an organization owner or admin reads the organization’s Knowledge settings, so ${KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_SETTINGS_FILE} is not pulled`
+
+/** A note this read adds to the whole fetch: kept out when a later fetch reads nothing of Knowledge. */
+export const isKnowledgeBaseNote = (note: string) => note === KNOWLEDGE_SIGN_IN || note === SETTINGS_UNREAD
+
+/** The settings' item, as the sync index and a change request name it; its file is `knowledge-base/settings.json`. */
+export const KNOWLEDGE_SETTINGS_NAME = 'settings'
+export const KNOWLEDGE_SETTINGS_REF = `${KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_SETTINGS_NAME}`
+
+interface BaseRow {
+  id: string
+  ownerWorkspaceId: string | null
+  name: string
+  description?: string | null
+  createdAt?: string
+  sharing?: { allWorkspaces: boolean; workspaceIds: string[] } | null
+}
+
+/** A base and the folder it is filed under. */
+interface FiledBase {
+  row: BaseRow
+  folder: string
+  /** Another base took the folder its name gives. */
+  renamed: boolean
+}
+
+/**
+ * What the Knowledge app gives one read, decided once: it does not run (404,
+ * nothing to say), the platform does not take knowledge bases from a project
+ * (nothing read, the project's left as they are), it runs and no one is
+ * signed in, a read failed (said as a warning, the last read kept), or the
+ * bases the linked workspace reads, each with its folder.
+ */
+export type KnowledgeReach =
+  | { state: 'off' }
+  | { state: 'not-taken' }
+  | { state: 'signed-out' }
+  | { state: 'unread' }
+  | { state: 'read'; person: PersonClients; workspaceId: string; bases: FiledBase[] }
+
+const errorCode = (err: unknown) => (err as { code?: string }).code
+
+/** The folder each base the lock holds was pulled into, by the base's id. */
+export function lockedFolders(root: string): Map<string, string> {
+  const out = new Map<string, string>()
+  for (const [ref, entry] of Object.entries(readLock(root)?.items ?? {})) {
+    if (ref.startsWith(`${KNOWLEDGE_BASE_DIR}/`) && ref !== KNOWLEDGE_SETTINGS_REF && entry.id) out.set(entry.id, ref.slice(KNOWLEDGE_BASE_DIR.length + 1))
+  }
+  return out
+}
+
+/** A name as the platform files it (`refName` in the service's sync index): as it is when already a reference name, else its words as a slug. */
+export const platformRefName = (name: string) => (/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) ? name : name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 24) || 'unnamed')
+
+/**
+ * The names the platform gives rows by id (`themeNames` in the service's sync
+ * index, which `knowledgeBaseNames` and `knowledgeSourceNames` use): by name,
+ * then id, each its reference name, numbered `-2` and on after one taken or reserved.
+ */
+export function platformNames(rows: ReadonlyArray<{ id: string; name: string }>, reserved: readonly string[] = []): Map<string, string> {
+  const out = new Map<string, string>()
+  const taken = new Set(reserved)
+  for (const row of [...rows].sort((a, b) => a.name.localeCompare(b.name) || a.id.localeCompare(b.id))) {
+    const base = platformRefName(row.name)
+    let name = base
+    for (let n = 2; taken.has(name); n++) name = `${base}-${n}`
+    taken.add(name)
+    out.set(row.id, name)
+  }
+  return out
+}
+
+/**
+ * The folder each knowledge base is filed under, by id. One the project holds
+ * already (`held`: the lock's, or one an Apply just created) keeps its folder,
+ * so a rename or a new base on the platform never moves a folder. Any other
+ * takes its name on the platform (the sync index's), numbered `-2` and on when
+ * a kept folder or `taken` has it. Folders are compared without case, as most
+ * computers compare them, and `settings` is the settings file's.
+ */
+export function knowledgeBaseFolders(platform: ReadonlyMap<string, string>, held: ReadonlyMap<string, string>, taken: Iterable<string> = []): Map<string, string> {
+  const pattern = new RegExp(KNOWLEDGE_BASE_NAME_PATTERN)
+  const used = new Set([KNOWLEDGE_SETTINGS_NAME, ...taken].map((f) => f.toLowerCase()))
+  const out = new Map<string, string>()
+  const ordered = [...platform].sort(([ia, a], [ib, b]) => a.localeCompare(b) || (ia < ib ? -1 : ia > ib ? 1 : 0))
+  for (const [id] of ordered) {
+    const folder = held.get(id)
+    if (!folder || !pattern.test(folder) || used.has(folder.toLowerCase())) continue
+    used.add(folder.toLowerCase())
+    out.set(id, folder)
+  }
+  for (const [id, name] of ordered) {
+    if (out.has(id)) continue
+    let folder = name
+    for (let n = 2; used.has(folder.toLowerCase()); n++) folder = `${name}-${n}`
+    used.add(folder.toLowerCase())
+    out.set(id, folder)
+  }
+  return out
+}
+
+/** Folders by base, as `knowledgeBaseFolders` files them; `held` may name every one (a fetch that read the index). */
+function fileBases(bases: BaseRow[], held: ReadonlyMap<string, string>): FiledBase[] {
+  const names = platformNames(bases, [KNOWLEDGE_SETTINGS_NAME])
+  const folders = knowledgeBaseFolders(names, held)
+  return bases.map((row) => {
+    const folder = folders.get(row.id)!
+    return { row, folder, renamed: !held.has(row.id) && folder.toLowerCase() !== platformRefName(row.name).toLowerCase() }
+  })
+}
+
+/** `capabilities` are the platform's sync features, read from it when not given. */
+export async function knowledgeReach(source: SyncSource, held: ReadonlyMap<string, string> = new Map(), warnings: string[] = [], capabilities?: readonly string[]): Promise<KnowledgeReach> {
+  const features = capabilities ?? (await source.workspace.find<{ features?: string[] }>('/v1/sync/capabilities').catch(() => null))?.features ?? []
+  if (!features.includes(KNOWLEDGE_BASES_FEATURE)) return { state: 'not-taken' }
+  const person = source.person
+  // A key is refused by the app but still finds it running: only a 404 means there is none.
+  if (!person) return (await source.workspace.find(`${APP}/me`).then((found) => found !== null, (err) => errorCode(err) !== 'NOT_FOUND')) ? { state: 'signed-out' } : { state: 'off' }
+  try {
+    if ((await person.workspace.find(`${APP}/me`)) === null) return { state: 'off' }
+    const listed = await person.workspace.get<{ bases?: BaseRow[] } | null>(`${APP}/bases`)
+    // The workspace's own bases and the organization's shared with it; never another workspace's.
+    const bases = (listed?.bases ?? []).filter((b) => b.ownerWorkspaceId === null || b.ownerWorkspaceId === source.workspaceId)
+    return { state: 'read', person, workspaceId: source.workspaceId, bases: fileBases(bases, held) }
+  } catch (err) {
+    if (errorCode(err) === 'UNAUTHORIZED') return { state: 'signed-out' }
+    warnings.push(`Knowledge bases: ${(err as Error).message}`)
+    return { state: 'unread' }
+  }
+}
+
+interface WorkspaceRow { id: string; slug?: string | null }
+
+/** Workspace slugs by id: every workspace for an administrator, else the person's own. */
+async function workspaceSlugs(org: SyncClient): Promise<Map<string, string>> {
+  const list = await org.get<WorkspaceRow[]>('/v1/workspaces?scope=org').catch((err) => {
+    if (errorCode(err) !== 'FORBIDDEN') throw err
+    return org.get<WorkspaceRow[]>('/v1/workspaces')
+  })
+  return new Map((Array.isArray(list) ? list : []).flatMap((w) => (w.slug ? [[w.id, w.slug] as const] : [])))
+}
+
+/** Slugs for workspace ids, sorted; an id with no slug the person can read stays an id, said in `unnamed`. */
+function slugsFor(ids: string[], slugs: ReadonlyMap<string, string>): { names: string[]; unnamed: string[] } {
+  const unnamed = ids.filter((id) => !slugs.has(id))
+  return { names: ids.map((id) => slugs.get(id) ?? id).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)), unnamed }
+}
+
+const unnamedNote = (ids: string[]) =>
+  ids.length === 1
+    ? `Shared with a workspace you can’t see by its slug, so it is written by its id (${ids[0]})`
+    : `Shared with ${ids.length} workspaces you can’t see by their slugs, so they are written by id (${ids.join(', ')})`
+
+interface SourceRow { id: string; kind: string; name: string; state?: string; schedule?: string; keepRemovedDays?: number; config?: Record<string, unknown>; auth?: { kind: string } & Record<string, unknown> }
+
+/** A secret's name for a source's credential: `KNOWLEDGE_<FOLDER>_<SOURCE>`, as `frontera secret set` takes it. */
+const credentialSecret = (folder: string, source: string) => `KNOWLEDGE_${folder}_${source}`.toUpperCase().replace(/[^A-Z0-9]+/g, '_').replace(/_+$/, '')
+
+const json = (value: unknown) => `${JSON.stringify(value, null, 2)}\n`
+
+async function readBase(reach: Extract<KnowledgeReach, { state: 'read' }>, filed: FiledBase, slugs: () => Promise<Map<string, string>>): Promise<RemoteItem> {
+  const { row, folder } = filed
+  const dir = `${KNOWLEDGE_BASE_DIR}/${folder}`
+  const notes: string[] = []
+  if (filed.renamed) notes.push(`Another knowledge base has the same name, so this one is in ${dir}`)
+  let doc: KnowledgeBaseFile
+  const named = { name: row.name, ...(row.description ? { description: row.description } : {}) }
+  if (row.ownerWorkspaceId !== null) doc = { ...named, scope: 'workspace' }
+  else {
+    let sharing: KnowledgeBaseSharing = { allWorkspaces: true }
+    if (!row.sharing?.allWorkspaces) {
+      const { names, unnamed } = slugsFor(row.sharing?.workspaceIds ?? [], await slugs())
+      if (unnamed.length) notes.push(unnamedNote(unnamed))
+      sharing = { workspaces: names }
+    }
+    doc = { ...named, scope: 'organization', sharing }
+  }
+  const files: Record<string, string> = { [`${dir}/${KNOWLEDGE_BASE_FILE}`]: json(doc) }
+  // An organization base's sources are managed at the organization, by a Knowledge steward.
+  const reader = row.ownerWorkspaceId === null ? reach.person.organization : reach.person.workspace
+  const listed = await reader.get<{ sources?: SourceRow[] } | null>(`${APP}/bases/${encodeURIComponent(row.id)}/sources`)
+  // A source's file is named as the platform knows it (`knowledgeSourceNames`): a change request matches sources by it.
+  const sources = (listed?.sources ?? []).filter((source) => source.state !== 'removing')
+  const keys = platformNames(sources)
+  const taken = new Set<string>()
+  const kept: string[] = []
+  for (const source of [...sources].sort((a, b) => (keys.get(a.id)! < keys.get(b.id)! ? -1 : 1))) {
+    const name = keys.get(source.id)!
+    if (source.kind !== 'api_pull') {
+      notes.push(`Its source “${source.name}” is set on the platform, so it is not pulled`)
+      kept.push(source.name)
+      continue
+    }
+    if (!source.config || !source.auth) {
+      const who = row.ownerWorkspaceId === null ? 'a Knowledge steward' : 'someone who may change this knowledge base'
+      notes.push(`Only ${who} reads the settings of its source “${source.name}”, so it stays on the platform and is not in ${KNOWLEDGE_BASE_SOURCES_DIR}/`)
+      kept.push(source.name)
+      continue
+    }
+    if (taken.has(name.toLowerCase())) {
+      notes.push(`Its source “${source.name}” is named like another but for capital letters, which one folder can’t hold, so it stays on the platform and is not in ${KNOWLEDGE_BASE_SOURCES_DIR}/`)
+      kept.push(source.name)
+      continue
+    }
+    taken.add(name.toLowerCase())
+    const secret = source.auth.kind === 'none' ? null : credentialSecret(folder, name)
+    files[`${dir}/${KNOWLEDGE_BASE_SOURCES_DIR}/${name}.json`] = json({
+      kind: source.kind,
+      name: source.name,
+      schedule: source.schedule,
+      keepRemovedDays: source.keepRemovedDays,
+      config: source.config,
+      auth: source.auth,
+      ...(secret ? { credentialSecret: secret } : {}),
+    })
+    if (secret) notes.push(`Its source “${source.name}” signs in with a credential, which never leaves the platform: set it on this computer with \`frontera secret set ${secret}\``)
+  }
+  notes.push(`Its documents stay on the platform unless asked for: pull with \`--documents ${dir}\``)
+  return { ref: dir, kind: 'knowledge-base', name: folder, title: row.name, id: row.id, files, unresolved: [], notes, requires: [], ...(kept.length ? { keptOnPlatform: kept } : {}) }
+}
+
+/** One setting the person may not read (403), as absent. */
+const unlessRefused = <T>(read: Promise<T>): Promise<T | null> => read.catch((err) => (errorCode(err) === 'FORBIDDEN' ? null : Promise.reject(err)))
+
+async function readSettings(reach: Extract<KnowledgeReach, { state: 'read' }>, slugs: () => Promise<Map<string, string>>, notes: string[]): Promise<RemoteItem | null> {
+  const org = reach.person.organization
+  const [assistant, embedding, policy] = await Promise.all([
+    unlessRefused(org.get<{ setup?: { enabled: boolean; model: { provider: string; modelId: string } | null; name: string | null; instructions: string | null } | null } | null>(`${APP}/assistant`)),
+    unlessRefused(org.get<{ chosen?: { provider: string; model: string } | null } | null>(`${APP}/settings/embedding`)),
+    unlessRefused(org.get<{ policy?: { apiSources: string; workspaceIds?: string[] } } | null>(`${APP}/source-policy`)),
+  ])
+  const doc: KnowledgeBaseSettingsFile = {}
+  const itemNotes: string[] = []
+  const setup = assistant?.setup
+  if (setup) doc.assistant = { enabled: setup.enabled, model: setup.model ? { provider: setup.model.provider, modelId: setup.model.modelId } : null, name: setup.name, instructions: setup.instructions }
+  if (embedding) doc.searchModel = embedding.chosen ? { provider: embedding.chosen.provider, model: embedding.chosen.model } : null
+  if (policy?.policy) {
+    const mode = policy.policy.apiSources
+    if (mode === 'chosen_workspaces') {
+      const { names, unnamed } = slugsFor(policy.policy.workspaceIds ?? [], await slugs())
+      if (unnamed.length) itemNotes.push(`Its source policy names ${unnamedNote(unnamed).slice('Shared with '.length)}`)
+      doc.sources = { apiSources: 'chosen_workspaces', workspaces: names }
+    } else doc.sources = { apiSources: mode } as KnowledgeSourcePolicy
+  }
+  if (!Object.keys(doc).length) {
+    notes.push(SETTINGS_UNREAD)
+    return null
+  }
+  const left = [!setup && 'the assistant', !embedding && 'the search model', !policy?.policy && 'the source policy'].filter((w): w is string => !!w)
+  if (left.length) itemNotes.push(`Left out, as only an organization owner or admin reads ${left.length === 1 ? 'it' : 'them'}: ${left.join(', ')}; applying the file leaves ${left.length === 1 ? 'it' : 'them'} as on the platform`)
+  return { ref: KNOWLEDGE_SETTINGS_REF, kind: 'knowledge-base', name: KNOWLEDGE_SETTINGS_NAME, title: 'Knowledge settings', files: { [`${KNOWLEDGE_BASE_DIR}/${KNOWLEDGE_BASE_SETTINGS_FILE}`]: json(doc) }, unresolved: [], notes: itemNotes, requires: [] }
+}
+
+export interface KnowledgeBasesRead {
+  items: RemoteItem[]
+  /** Said once for the whole read: a sign-in to make, settings the person may not read. */
+  notes: string[]
+  /** The items are everything there is (the app is off, or every base was read): what is not among them is gone. */
+  listed: boolean
+}
+
+/**
+ * The bases and settings: every one, or those `only` names (a fetch reads
+ * what the sync index says moved). One that fails leaves the read unlisted, so
+ * nothing reads as deleted.
+ */
+export async function readKnowledgeBases(reach: KnowledgeReach, warnings: string[], only?: ReadonlySet<string>): Promise<KnowledgeBasesRead> {
+  if (reach.state === 'off') return { items: [], notes: [], listed: true }
+  if (reach.state === 'signed-out') return { items: [], notes: [KNOWLEDGE_SIGN_IN], listed: false }
+  if (reach.state === 'unread' || reach.state === 'not-taken') return { items: [], notes: [], listed: false }
+  const org = reach.person.organization
+  let known: Promise<Map<string, string>> | null = null
+  const slugs = () => (known ??= workspaceSlugs(org))
+  const notes: string[] = []
+  try {
+    const wanted = (ref: string) => !only || only.has(ref)
+    const bases = await Promise.all(reach.bases.filter((b) => wanted(`${KNOWLEDGE_BASE_DIR}/${b.folder}`)).map((b) => readBase(reach, b, slugs)))
+    const settings = wanted(KNOWLEDGE_SETTINGS_REF) ? await readSettings(reach, slugs, notes) : null
+    return { items: [...bases, ...(settings ? [settings] : [])], notes, listed: !only }
+  } catch (err) {
+    warnings.push(`Knowledge bases: ${(err as Error).message}`)
+    return { items: [], notes: [], listed: false }
+  }
+}
+
+interface PickRow { id: string; name: string; picked: boolean; unavailable?: string | null }
+
+/** An Agent's picks as agent.json names them (`knowledge-base/<folder>`), or null when the bases were not read. */
+export async function agentPicks(reach: KnowledgeReach, agentId: string): Promise<{ refs: KnowledgeBaseRef[]; notes: string[] } | null> {
+  if (reach.state !== 'read') return null
+  const answer = await reach.person.workspace.get<{ bases?: PickRow[] } | null>(`${APP}/agents/${encodeURIComponent(agentId)}/bases`)
+  const folders = new Map(reach.bases.map((b) => [b.row.id, b.folder]))
+  const refs: KnowledgeBaseRef[] = []
+  const notes: string[] = []
+  for (const pick of (answer?.bases ?? []).filter((b) => b.picked)) {
+    const folder = pick.unavailable ? undefined : folders.get(pick.id)
+    if (folder) refs.push(`${KNOWLEDGE_BASE_DIR}/${folder}`)
+    else notes.push(`Also searches the knowledge base “${pick.name}”, which this workspace no longer reads: it is not in knowledgeBases, so applying agent.json drops it`)
+  }
+  return { refs: refs.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)), notes }
+}
+
+// ── Documents, when asked for ────────────────────────────────────────────────
+
+const DOCUMENTS_DIR = 'documents'
+
+/**
+ * The documents in `knowledge-base/<folder>/documents/`, by file name: what a
+ * push may bring. A knowledge base has no folders, so a file in a subfolder is
+ * named in `nested` and stays here.
+ */
+export function knowledgeBaseDocuments(root: string, folder: string): { documents: Array<{ fileName: string; bytes: Uint8Array }>; nested: string[] } {
+  const dir = join(root, KNOWLEDGE_BASE_DIR, folder, DOCUMENTS_DIR)
+  const out: { documents: Array<{ fileName: string; bytes: Uint8Array }>; nested: string[] } = { documents: [], nested: [] }
+  let names: string[]
+  try {
+    names = readdirSync(dir).sort()
+  } catch {
+    return out
+  }
+  for (const name of names) {
+    if (name.startsWith('.') || name.endsWith('.workspace')) continue
+    const st = statSync(join(dir, name), { throwIfNoEntry: false })
+    if (st?.isDirectory()) out.nested.push(name)
+    else if (st?.isFile()) out.documents.push({ fileName: name, bytes: new Uint8Array(readFileSync(join(dir, name))) })
+  }
+  return out
+}
+const IGNORED = `${KNOWLEDGE_BASE_DIR}/*/${DOCUMENTS_DIR}/`
+const IGNORE_COMMENT = '# Knowledge base documents: brought on request, often the customer’s own content'
+
+/** Add the documents line to the project's `.gitignore`, appended and only once. */
+export function ignoreKnowledgeBaseDocuments(root: string): void {
+  const path = join(root, '.gitignore')
+  const current = existsSync(path) ? readFileSync(path, 'utf8') : ''
+  if (current.split('\n').some((line) => [IGNORED, `/${IGNORED}`].includes(line.trim()))) return
+  const lead = current ? `${current}${current.endsWith('\n') ? '' : '\n'}\n` : ''
+  writeFileAtomic(path, `${lead}${IGNORE_COMMENT}\n${IGNORED}\n`, existsSync(path) ? statSync(path).mode & 0o777 : 0o644)
+}
+
+interface DocumentRow { id: string; title?: string | null; fileName?: string | null; createdAt?: string | null; source?: { id: string; name: string } | null }
+
+/** A document's name as a file: no path, no leading dot. */
+const fileNameOf = (doc: DocumentRow) => (doc.fileName ?? doc.title ?? '').replace(/\\/g, '/').split('/').pop()!.replace(/^\.+/, '').trim() || null
+
+/** A base's documents as the workspace has them now, each by its file name: every one, synced ones included. */
+export async function knowledgeBaseDocumentsOn(client: SyncClient, baseId: string): Promise<Array<{ id: string; fileName: string; createdAt: string | null; synced: boolean }>> {
+  const documents = (await client.get<{ documents?: DocumentRow[] } | null>(`${APP}/bases/${encodeURIComponent(baseId)}/documents`))?.documents ?? []
+  return documents.flatMap((d) => {
+    const fileName = fileNameOf(d)
+    return fileName ? [{ id: d.id, fileName, createdAt: d.createdAt ?? null, synced: !!d.source }] : []
+  })
+}
+
+/** `lembur.md` → `lembur-2.md`. */
+function numbered(name: string, n: number): string {
+  const dot = name.lastIndexOf('.')
+  return dot > 0 ? `${name.slice(0, dot)}-${n}${name.slice(dot)}` : `${name}-${n}`
+}
+
+/**
+ * Bring a base's uploaded documents into `knowledge-base/<folder>/documents/`.
+ * A document a source synced stays with that source (`skipped`). A file already in the
+ * folder is never overwritten. Each written document's id and digest are
+ * recorded (`.frontera/sync/documents.json`, under `knowledge-base/<folder>`),
+ * so a push tells one changed here from one new here.
+ */
+export async function pullKnowledgeBaseDocuments(root: string, client: SyncClient, base: { folder: string; id: string }): Promise<DocumentsPulled> {
+  const out: DocumentsPulled = { written: [], present: [], skipped: [], failed: [] }
+  const key = `${KNOWLEDGE_BASE_DIR}/${base.folder}`
+  const dir = `${key}/${DOCUMENTS_DIR}`
+  let documents: DocumentRow[]
+  try {
+    documents = (await client.get<{ documents?: DocumentRow[] } | null>(`${APP}/bases/${encodeURIComponent(base.id)}/documents`))?.documents ?? []
+  } catch (err) {
+    out.failed.push(`the list of documents: ${(err as Error).message}`)
+    return out
+  }
+  const workspace = readLink(root)?.workspace
+  // A document pulled before keeps its file name, whatever order the platform lists it in now.
+  const before = new Map(Object.entries(workspace ? (readPulledDocuments(root, workspace).bases[key] ?? {}) : {}).map(([path, d]) => [d.sourceId, path]))
+  const taken = new Set([...before.values()].map((p) => p.toLowerCase()))
+  const recorded: Record<string, PulledDocument> = {}
+  for (const doc of documents) {
+    const name = fileNameOf(doc)
+    if (!name) continue
+    if (doc.source) {
+      out.skipped.push(name)
+      continue
+    }
+    let rel = before.get(doc.id)
+    if (!rel) {
+      rel = name
+      for (let n = 2; taken.has(rel.toLowerCase()); n++) rel = numbered(name, n)
+      taken.add(rel.toLowerCase())
+    }
+    const path = `${dir}/${rel}`
+    if (existsSync(join(root, path))) {
+      out.present.push(path)
+      continue
+    }
+    try {
+      const bytes = await client.getBytes(`${APP}/documents/${encodeURIComponent(doc.id)}/file`)
+      mkdirSync(dirname(join(root, path)), { recursive: true })
+      writeFileSync(join(root, path), bytes)
+      out.written.push(path)
+      recorded[rel] = { sourceId: doc.id, sha256: sha256Of(bytes), bytes: bytes.byteLength, at: new Date().toISOString() }
+    } catch (err) {
+      out.failed.push(`${name}: ${(err as Error).message}`)
+    }
+  }
+  if (out.written.length) ignoreKnowledgeBaseDocuments(root)
+  if (workspace && Object.keys(recorded).length) recordPulledDocuments(root, workspace, key, recorded)
+  return out
+}