@frontera-sdk/cli 1.53.23 → 1.53.24

package/src/local/local-studio-server.ts

@@ -10,7 +10,7 @@ import { projectBlueprintOutline } from './local-blueprint-outline'
 import { currentBuildRows, currentBuildSummary } from './local-build-inputs'
 import { appStatusRow, itemBuild } from './local-build-summary'
 import { KnowledgeDocumentError, KNOWLEDGE_UPLOAD_EXTENSIONS, MAX_KNOWLEDGE_UPLOAD_BYTES, checkUpload, documentInFolder, knowledgeFolderOf, projectDocuments, removeKnowledgeDocument, removeRuntimeCopies, runtimeDocuments, saveKnowledgeDocument, type ServiceCall } from './local-knowledge-uploads'
-import { isAlive, logPath, readApplied, readState, type ComponentName } from './local-state'
+import { COMPONENTS, isAlive, logPath, readApplied, readState, type ComponentName } from './local-state'
 import { appendTestCase, decisionOfApprovals, MIXED_DECISIONS_LINE, noteRedactedVariables, redactedVariablesLine, redactedVariablesOfTimeline, renderTestCase, variablesOfTimeline } from './local-test-cases'
 import { findCaseFiles } from './local-test-runner'
 import { workflowTestListing } from './local-workflow-tests'
@@ -91,7 +91,7 @@ function cookieToken(req: Request): string | null {
   return null
 }
 
-const LOG_COMPONENTS: ComponentName[] = ['postgres', 'storage', 'service', 'jobs', 'runner', 'studio']
+const LOG_COMPONENTS: readonly ComponentName[] = COMPONENTS
 
 function tail(path: string, lines: number): string[] {
   if (!existsSync(path)) return []
@@ -193,7 +193,7 @@ export function startStudioServer(opts: StudioServerOptions) {
       const run = testRuns.get(runMatch[1]!)
       return run ? json(run) : json({ error: 'no such test run' }, 404)
     }
-    const logMatch = /^\/__local\/logs\/([a-z]+)$/.exec(url.pathname)
+    const logMatch = /^\/__local\/logs\/([a-z-]+)$/.exec(url.pathname)
     if (logMatch) {
       const component = logMatch[1] as ComponentName
       if (!LOG_COMPONENTS.includes(component)) return json({ error: 'unknown component' }, 404)

package/src/local/local-supervisor.ts

@@ -6,6 +6,7 @@ import {
   containerState,
   copyFromContainer,
   dbVolumeName,
+  dockerMemoryBytes,
   execInContainer,
   followLogs,
   imagePresent,
@@ -49,8 +50,10 @@ import { recordRuntime } from './local-runtime-history'
 import { recordExit } from '../telemetry/cli-telemetry'
 import { buildDocs, checkoutDocsStale, copyDocs, customerDocsBundle, IMAGE_DOCS_PATH, writeDocsBuildMarker } from './local-docs'
 import { keepSchemas, type SchemaSource } from './local-schemas'
-import { applyModelKeys, ensureBlueprint, ensureOrgKey, ensureProviderModels, retireLocalModel, seedLocalRuntime, signIn } from './local-seed'
+import { applyModelKeys, ensureBlueprint, ensureKnowledgeSteward, ensureOrgKey, ensureProviderModels, retireLocalModel, seedLocalRuntime, signIn } from './local-seed'
+import { SEARCH_INDEX_IMAGE, searchIndexChoice, searchIndexRunSpec, waitForSearchIndex } from './local-search-index'
 import {
+  COMPONENTS,
   logPath,
   readState,
   updateComponent,
@@ -93,7 +96,8 @@ export const containerName = (project: LocalProject, component: ComponentName) =
 export const volumeName = (project: LocalProject) => dbVolumeName(project.id)
 
 interface Running {
-  children: Map<ComponentName, ChildProcess>
+  // The search index is a container, never a child process.
+  children: Map<Exclude<ComponentName, 'search-index'>, ChildProcess>
   containers: Set<string>
   studio?: ReturnType<typeof startStudioServer>
   embeddings?: ReturnType<typeof startLocalEmbeddings>
@@ -105,7 +109,8 @@ export async function runSupervisor(project: LocalProject, ports: PortBlock, ses
   if (!config) throw new Error('no local runtime config; run `frontera local start`')
   const secrets = loadOrCreateSecrets(project)
   const previous = readState(project)
-  const enabled = (c: ComponentName) => !config.exclude.includes(c)
+  const searchIndex = searchIndexChoice({ root: project.root, with: config.with ?? [], exclude: config.exclude, dockerMemoryBytes: dockerMemoryBytes() })
+  const enabled = (c: ComponentName) => (c === 'search-index' ? searchIndex.on : !config.exclude.includes(c))
 
   let state: LocalState = {
     version: 1,
@@ -118,13 +123,13 @@ export async function runSupervisor(project: LocalProject, ports: PortBlock, ses
     startedAt: new Date().toISOString(),
     updatedAt: new Date().toISOString(),
     ports,
-    components: (['postgres', 'storage', 'service', 'jobs', 'runner', 'studio'] as ComponentName[]).map(
+    components: COMPONENTS.map(
       (name): ComponentState => ({
         name,
         status: enabled(name) ? 'starting' : 'off',
-        kind: name === 'studio' ? 'supervisor' : name === 'postgres' || name === 'jobs' || name === 'storage' ? 'container' : config.mode === 'source' ? 'process' : 'container',
+        kind: name === 'studio' ? 'supervisor' : name === 'service' || name === 'runner' ? (config.mode === 'source' ? 'process' : 'container') : 'container',
         port: ports[name],
-        detail: enabled(name) ? undefined : 'excluded',
+        detail: enabled(name) ? undefined : name === 'search-index' && !searchIndex.on ? searchIndex.why : 'excluded',
       }),
     ),
     credentials: previous?.credentials,
@@ -177,6 +182,11 @@ export async function runSupervisor(project: LocalProject, ports: PortBlock, ses
     await startPostgres(project, ports, secrets, running, set)
     // Object storage before anything that uploads to it.
     if (enabled('storage')) await startStorage(project, ports, secrets, running, set)
+    // The search index takes about a minute on one CPU, and the service reaches it only when Knowledge is used:
+    // it comes up while the rest starts, and the runtime is ready once it answers. It says its own failure.
+    const searchIndexFailed = enabled('search-index')
+      ? startSearchIndex(project, ports, running, set).then(() => null, (err: Error) => err.message)
+      : Promise.resolve(null)
     // 2. Job engine (before the service, so the service's registration lands)
     if (enabled('jobs')) await startJobs(project, ports, running, set)
     // Keyless embeddings answer before the service needs them.
@@ -203,7 +213,7 @@ export async function runSupervisor(project: LocalProject, ports: PortBlock, ses
       },
     })
     // 3. Service
-    await startService(project, ports, secrets, config, running, set, enabled('runner'), enabled('jobs'), enabled('storage'))
+    await startService(project, ports, secrets, config, running, set, enabled('runner'), enabled('jobs'), enabled('storage'), enabled('search-index'))
     // The service has migrated; the catalog it reads is seeded per environment.
     seedPluginCatalog(project)
     // 4. Function runner
@@ -269,10 +279,18 @@ export async function runSupervisor(project: LocalProject, ports: PortBlock, ses
       organizationId: state.credentials!.organizationId,
       logFile: logPath(project, 'service'),
     })
+    if (enabled('search-index')) {
+      await ensureKnowledgeSteward({ serviceUrl, password: secrets.ownerPassword, organizationId: state.credentials!.organizationId, logFile: logPath(project, 'service') })
+    }
     state = { ...state, credentials: { ...state.credentials!, session, modelProviders } }
     save()
     // The formats of the project's files, for the release the service runs.
     await keepFileSchemas(project, { serviceUrl, token: state.credentials!.workspaceKey ?? session }, logPath(project, 'service'))
+    const searchIndexProblem = await searchIndexFailed
+    if (searchIndexProblem) {
+      await fail('search-index', searchIndexProblem)
+      return
+    }
     // 6. Studio. From a checkout, rebuild it first when its sources changed
     // since the last build, so a pulled change is never served stale.
     if (studioBuildStale(config)) {
@@ -294,7 +312,8 @@ export async function runSupervisor(project: LocalProject, ports: PortBlock, ses
     save()
     recordRuntime(project, { event: 'started', at: new Date().toISOString(), since: state.startedAt })
   } catch (err) {
-    const failing = state.components.find((c) => c.status === 'starting')?.name ?? 'service'
+    // The search index starts beside the others and says its own failure above.
+    const failing = state.components.find((c) => c.status === 'starting' && c.name !== 'search-index')?.name ?? 'service'
     await fail(failing, (err as Error).message)
     return
   }
@@ -491,6 +510,22 @@ async function startStorage(project: LocalProject, ports: PortBlock, secrets: Lo
   throw new Error('object storage did not answer within 45 s')
 }
 
+async function startSearchIndex(project: LocalProject, ports: PortBlock, running: Running, set: SetFn) {
+  const name = containerName(project, 'search-index')
+  await ensureImage(SEARCH_INDEX_IMAGE, project, 'search-index')
+  const current = containerState(name)
+  if (current === 'exited') startContainer(name)
+  else if (current !== 'running') {
+    if (current !== 'missing') removeContainer(name)
+    runContainer(searchIndexRunSpec({ name, projectId: project.id, port: ports['search-index'] }))
+  }
+  running.containers.add(name)
+  followLogs(name, logPath(project, 'search-index'))
+  const why = await waitForSearchIndex(ports['search-index'], 180_000)
+  if (why) throw new Error(`the search index did not come up within 180 s (${why}); see \`frontera local logs search-index\`, or start without it: \`frontera local start --exclude search-index\``)
+  set('search-index', { status: 'running', container: name, startedAt: new Date().toISOString() })
+}
+
 async function startJobs(project: LocalProject, ports: PortBlock, running: Running, set: SetFn) {
   const name = containerName(project, 'jobs')
   await ensureImage(IMAGES.jobs, project, 'jobs')
@@ -527,6 +562,7 @@ async function startService(
   runnerEnabled: boolean,
   jobsEnabled: boolean,
   storageEnabled: boolean,
+  searchIndexEnabled: boolean,
 ) {
   const env = serviceEnv({
     secrets,
@@ -535,6 +571,7 @@ async function startService(
     runnerEnabled,
     jobsEnabled,
     storageEnabled,
+    searchIndexEnabled,
     keylessEmbeddings: localEmbeddingsReachable(),
     documentationBundle: config.mode === 'source' ? customerDocsBundle(config.platformSource) : null,
   })

package/src/local/local-sync-link.ts

@@ -12,6 +12,7 @@ import { configDir } from '../paths'
 import { CliError } from '../errors'
 import { deleteProfile, fingerprint, getProfile, putProfile } from '../profiles'
 import { syncClient, type SyncClient } from './local-sync-client'
+import type { PersonClients } from './local-sync-remote'
 import { PROJECT_FILES, type LocalProject } from './local-project'
 import { normalizeOrigin } from './local-sync-client'
 
@@ -360,6 +361,21 @@ export async function organizationReader(owner: KeyOwner, link: SyncLink): Promi
   return { client: byKey, notSteward: byKey ? null : (person.email ?? 'the person') }
 }
 
+/** One session's two places: the linked workspace, and its organization (where a steward or an administrator reads more). */
+export function personClients(platform: string, token: string, orgId: string, workspaceId: string): PersonClients {
+  return {
+    workspace: syncClient(platform, token, { 'x-org-id': orgId, 'x-workspace-id': workspaceId }),
+    organization: syncClient(platform, token, { 'x-org-id': orgId }),
+  }
+}
+
+/** The person signed in for the link or a named target, as the Knowledge app reads them; null with no session. */
+export async function personReader(owner: KeyOwner, link: SyncLink): Promise<PersonClients | null> {
+  const person = await targetSession(owner, link)
+  const orgId = person ? await targetOrganization(owner, link) : null
+  return person && orgId ? personClients(link.platform, person.token, orgId, link.workspace) : null
+}
+
 const organizationReads = new Map<string, Promise<boolean>>()
 
 /** Whether the person may read the organization's Blueprint, asked once per process: a refusal there is a refusal of every organization read. */

package/src/local/local-sync-names.ts

@@ -12,7 +12,8 @@ const KIND_WORD: Record<string, string> = {
   workflow: 'Workflow',
   function: 'Function',
   skill: 'Skill',
-  knowledge: 'Knowledge base',
+  knowledge: 'Agent knowledge',
+  'knowledge-base': 'Knowledge base',
   plugin: 'Plugin',
   evaluation: 'Evaluation dataset',
   dataset: 'Dataset',
@@ -44,6 +45,8 @@ export function readableName(slug: string): string {
 export function kindWord(ref: string): string {
   const [kind, sub] = ref.split('/')
   if (kind === 'blueprint') return BLUEPRINT_WORD[sub ?? ''] ?? 'Blueprint'
+  // The organization's Knowledge settings live in the knowledge-base folder but are no base.
+  if (ref === 'knowledge-base/settings') return 'Knowledge settings'
   return KIND_WORD[kind ?? ''] ?? readableName(kind ?? 'item')
 }
 
@@ -114,6 +117,7 @@ export function namingPaths(kind: string, name: string): string[] {
   if (kind === 'chat-app') return [`chat-apps/${name}.json`]
   if (kind === 'app') return [`apps/${name}.json`]
   if (kind === 'knowledge') return [`knowledge/${name}/knowledge.json`]
+  if (kind === 'knowledge-base') return name === 'settings' ? [] : [`knowledge-base/${name}/knowledge-base.json`]
   if (kind === 'dataset') return [`datasets/${name}.json`]
   return []
 }
@@ -160,6 +164,11 @@ export function itemTitle(kind: string, name: string, files: Record<string, stri
     const own = declared(parseJson(files[`knowledge/${name}/knowledge.json`]), 'displayName', 'name')
     if (own) return own
   }
+  if (kind === 'knowledge-base') {
+    if (name === 'settings') return 'Knowledge settings'
+    const own = declared(parseJson(files[`knowledge-base/${name}/knowledge-base.json`]), 'name')
+    if (own) return own
+  }
   // An Agent's lessons are named by their Agent: "Lessons for Agent Claims desk".
   if (kind === 'lessons') {
     const own = declared((parseJson(files[`agents/${name}/agent.json`]) as { config?: unknown } | null)?.config, 'name')

package/src/local/local-sync-plan.ts

@@ -1,6 +1,6 @@
 import { createHash } from 'node:crypto'
-import { existsSync, mkdirSync, readdirSync, readFileSync, rmdirSync, rmSync, writeFileSync } from 'node:fs'
-import { dirname, join, relative, resolve, sep } from 'node:path'
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmdirSync, rmSync, writeFileSync } from 'node:fs'
+import { basename, dirname, join, relative, resolve, sep } from 'node:path'
 
 import { parse as parseYaml } from 'yaml'
 
@@ -28,11 +28,16 @@ import { comparablePortable, toLocal } from './local-sync-portable'
  * pull produces a `dataset/` item. It stays a kind so `--kinds datasets`, a
  * lock written with one and the change-request kinds keep reading as before.
  *
+ * `knowledge-base` is the Knowledge app's (`knowledge-base/<folder>/`, and
+ * the organization's `knowledge-base/settings.json`), apart from Agent
+ * knowledge (`knowledge`). A base's documents are not in its item: they come
+ * only when asked for (`pull --documents knowledge-base/<folder>`).
+ *
  * `lessons` is an Agent's `lessons.yaml` as a change request carries it
  * (`lessons/<slug>`): brought by `pull --lessons`, never listed by a fetch or
  * kept in the lock, and proposed back as rules for the platform's reviewers.
  */
-export const SYNC_KINDS = ['blueprint', 'dataset', 'skill', 'function', 'plugin', 'knowledge', 'workflow', 'agent', 'evaluation', 'theme', 'chat-app', 'app', 'brand', 'library', 'lessons'] as const
+export const SYNC_KINDS = ['blueprint', 'dataset', 'skill', 'function', 'plugin', 'knowledge', 'knowledge-base', 'workflow', 'agent', 'evaluation', 'theme', 'chat-app', 'app', 'brand', 'library', 'lessons'] as const
 export type SyncKind = (typeof SYNC_KINDS)[number]
 
 export const KIND_LABEL: Record<SyncKind, string> = {
@@ -42,6 +47,7 @@ export const KIND_LABEL: Record<SyncKind, string> = {
   function: 'Functions',
   plugin: 'Plugins',
   knowledge: 'Knowledge',
+  'knowledge-base': 'Knowledge bases',
   workflow: 'Workflows',
   agent: 'Agents',
   evaluation: 'Evaluations',
@@ -93,6 +99,18 @@ export interface RemoteItem {
    * for the original source.
    */
   recovered?: string[]
+  /**
+   * What the platform keeps of it that its files leave out: a knowledge
+   * base's sources the read could not file, by name. A change request carries
+   * every source a base has, so a push never proposes it from these files.
+   */
+  keptOnPlatform?: string[]
+  /**
+   * Its name in the workspace's sync index, when the project files it under
+   * another (a knowledge base keeps the folder it was pulled into): what a
+   * change request names it by.
+   */
+  platformName?: string
 }
 
 export interface ItemAsset {
@@ -538,11 +556,31 @@ export function entryFor(item: RemoteItem, now: string): LockEntry {
 
 export const WORKSPACE_SUFFIX = '.workspace'
 
+type CaseFs = { existsSync(path: string): boolean; readdirSync(path: string): string[]; renameSync(from: string, to: string): void }
+
+/**
+ * Where the file system ignores case (a Mac's, by default), writing
+ * `Newsroom.json` over `newsroom.json` keeps the old name, and a push matches a
+ * knowledge base's sources by file name: the file there is renamed to the
+ * workspace's name first. Where case counts, they are two files, and both stay.
+ * True when it renamed one.
+ */
+export function matchCase(full: string, fs: CaseFs = { existsSync, readdirSync, renameSync }): boolean {
+  if (!fs.existsSync(full)) return false
+  const name = basename(full)
+  const entries = fs.readdirSync(dirname(full))
+  const held = entries.includes(name) ? undefined : entries.find((e) => e.toLowerCase() === name.toLowerCase())
+  if (!held) return false
+  fs.renameSync(join(dirname(full), held), full)
+  return true
+}
+
 /** Write or delete a working file; true when it changed. */
 function put(root: string, path: string, content: string | null): boolean {
   const full = safePath(root, path)
+  const renamed = content !== null && matchCase(full)
   const now = readLocal(root, path)
-  if (now === content) return false
+  if (now === content) return renamed
   if (content === null) rmSync(full, { force: true })
   else {
     mkdirSync(dirname(full), { recursive: true })
@@ -628,11 +666,12 @@ export function applyItems(root: string, workspace: string, items: RemoteItem[],
       mkdirSync(dirname(full), { recursive: true })
       // A Plugin's secret values set here stay: they are this computer's configuration. So do a mapped model and
       // the multi-turn evaluation cases (`local-sync-portable.ts`).
+      const renamed = matchCase(full)
       const content = isPluginFile(path) ? withLocalSecrets(remote, readLocal(root, path)) : toLocal(path, remote, readLocal(root, path), projectModels(root), opts.base?.(path) ?? null)
       if (readLocal(root, path) !== content) {
         writeFileSync(full, content)
         written.push(path)
-      }
+      } else if (renamed) written.push(path)
     }
     for (const [path, wroteHash] of Object.entries(previous?.files ?? {})) {
       if (path in item.files) continue