@frontera-sdk/cli 1.52.39 → 1.52.41

package/src/local/local-knowledge-ingestion.ts

@@ -0,0 +1,225 @@
+import type { LocalCredentials } from './local-state'
+import { rows } from './local-sync-remote'
+
+/**
+ * Knowledge ingestion on the local runtime, around a database snapshot.
+ *
+ * A document is read by a background job: its row goes `pending`, then
+ * `processing`, then `ready` with its chunks (rows in the same database). A
+ * snapshot taken while a job runs holds the row as `processing` with no
+ * chunks, and the job's work lands in the database the restore drops: once
+ * that snapshot is restored, the row says `processing` forever and nothing
+ * is reading it.
+ *
+ * So `local check` waits for ingestion to settle before it keeps the
+ * database as applied and before its tests (`waitForIngestion`), and after a
+ * restore it queues every document the snapshot holds unsettled again
+ * (`requeueSources`: the platform's retry, which replaces a document's
+ * chunks, so a second read never duplicates them). `local status` and
+ * `local check` name a document still stuck (`stuckSources`), with the
+ * command that re-queues it.
+ */
+
+export interface IngestSource {
+  base: string
+  baseId: string
+  id: string
+  fileName: string
+  status: string
+  updatedAt?: string
+}
+
+/** The two calls these need, against the runtime as its workspace key. */
+export interface IngestApi {
+  get: (path: string) => Promise<unknown>
+  post: (path: string, body: unknown) => Promise<unknown>
+}
+
+/** A row in one of these is being read, or waits for its job. */
+export const UNSETTLED = new Set(['pending', 'processing'])
+
+/** A document unsettled for longer than this has no job behind it: ingestion of a large scan takes minutes, not this. */
+export const STUCK_AFTER_MS = 30 * 60_000
+
+/** The runtime's API as the workspace key; null without one. */
+export function localIngestApi(serviceUrl: string, creds: Pick<LocalCredentials, 'workspaceKey'>): IngestApi | null {
+  const key = creds.workspaceKey
+  if (!key) return null
+  const call = async (method: string, path: string, body?: unknown) => {
+    const res = await fetch(new URL(path, serviceUrl), {
+      method,
+      headers: { authorization: `Bearer ${key}`, ...(body === undefined ? {} : { 'content-type': 'application/json' }) },
+      ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+      signal: AbortSignal.timeout(10_000),
+    })
+    const text = await res.text()
+    if (!res.ok) throw new Error(`${res.status} ${text.slice(0, 200)}`)
+    return text ? (JSON.parse(text) as unknown) : null
+  }
+  return { get: (path) => call('GET', path), post: (path, body) => call('POST', path, body) }
+}
+
+/** Every document in every knowledge base of the workspace; null when the runtime did not answer. */
+export async function listIngestSources(api: IngestApi, workspaceId: string): Promise<IngestSource[] | null> {
+  try {
+    const bases = rows<{ id?: string; name?: string }>(await api.get(`/v1/workspace-knowledge?workspaceId=${encodeURIComponent(workspaceId)}`))
+    const out: IngestSource[] = []
+    for (const base of bases) {
+      if (!base.id) continue
+      const listed = rows<{ id?: string; fileName?: string; status?: string; updatedAt?: string }>(await api.get(`/v1/workspace-knowledge/${encodeURIComponent(base.id)}/sources`))
+      for (const s of listed) {
+        if (!s.id) continue
+        out.push({ base: base.name ?? base.id, baseId: base.id, id: s.id, fileName: s.fileName ?? s.id, status: String(s.status ?? ''), ...(s.updatedAt ? { updatedAt: s.updatedAt } : {}) })
+      }
+    }
+    return out
+  } catch {
+    return null
+  }
+}
+
+/**
+ * The documents unsettled with no job behind them: last changed before the
+ * database was restored (the row came from the snapshot; its job ran on the
+ * database the restore replaced), or not changed for `STUCK_AFTER_MS`.
+ */
+export function stuckSources(sources: IngestSource[], opts: { restoredAt?: string | null; now?: number; staleMs?: number } = {}): IngestSource[] {
+  const now = opts.now ?? Date.now()
+  const staleMs = opts.staleMs ?? STUCK_AFTER_MS
+  const restored = opts.restoredAt ? Date.parse(opts.restoredAt) : NaN
+  return sources.filter((s) => {
+    if (!UNSETTLED.has(s.status)) return false
+    const at = s.updatedAt ? Date.parse(s.updatedAt) : NaN
+    if (!Number.isFinite(at)) return false
+    return (Number.isFinite(restored) && at < restored) || now - at > staleMs
+  })
+}
+
+export interface IngestWait {
+  /** Nothing is pending or processing (or the runtime did not say). */
+  settled: boolean
+  /** What was still unsettled when the wait ended. */
+  waiting: IngestSource[]
+  /** What failed to read, said with its retry. */
+  failed: IngestSource[]
+  ms: number
+}
+
+/** Wait (bounded) until no document is pending or processing. */
+export async function waitForIngestion(
+  api: IngestApi,
+  workspaceId: string,
+  opts: { timeoutMs?: number; pollMs?: number; sleep?: (ms: number) => Promise<void>; now?: () => number } = {},
+): Promise<IngestWait> {
+  const now = opts.now ?? Date.now
+  const sleep = opts.sleep ?? ((ms: number) => Bun.sleep(ms))
+  const started = now()
+  const deadline = started + (opts.timeoutMs ?? 120_000)
+  for (;;) {
+    const sources = await listIngestSources(api, workspaceId)
+    // A runtime that does not answer is not waited on: the snapshot and the tests say what they say.
+    if (!sources) return { settled: true, waiting: [], failed: [], ms: now() - started }
+    const waiting = sources.filter((s) => UNSETTLED.has(s.status))
+    const failed = sources.filter((s) => s.status === 'failed')
+    if (!waiting.length || now() >= deadline) return { settled: !waiting.length, waiting, failed, ms: now() - started }
+    await sleep(opts.pollMs ?? 1000)
+  }
+}
+
+export interface Requeued {
+  requeued: IngestSource[]
+  refused: Array<{ source: IngestSource; reason: string }>
+}
+
+/** Each document queued to be read again: the platform's retry, which sets it pending and replaces its chunks. */
+export async function requeueSources(api: IngestApi, sources: IngestSource[]): Promise<Requeued> {
+  const out: Requeued = { requeued: [], refused: [] }
+  for (const s of sources) {
+    try {
+      await api.post(`/v1/workspace-knowledge/${encodeURIComponent(s.baseId)}/sources/${encodeURIComponent(s.id)}/retry`, {})
+      out.requeued.push(s)
+    } catch (err) {
+      out.refused.push({ source: s, reason: (err as Error).message })
+    }
+  }
+  return out
+}
+
+/** Every unsettled document after a restore: the snapshot holds it so, and its job is gone with the database it ran on. */
+export async function requeueAfterRestore(api: IngestApi, workspaceId: string): Promise<Requeued & { listed: boolean }> {
+  const sources = await listIngestSources(api, workspaceId)
+  if (!sources) return { listed: false, requeued: [], refused: [] }
+  return { listed: true, ...(await requeueSources(api, sources.filter((s) => UNSETTLED.has(s.status)))) }
+}
+
+/** A value as a shell word: quoted when it holds anything but plain characters. */
+export const shellArg = (v: string) => (/^[\w.@/-]+$/.test(v) ? v : `"${v.replace(/(["\\$`])/g, '\\$1')}"`)
+
+/** The command that re-queues one document. */
+export const retryCommand = (s: IngestSource) => `frontera knowledge retry ${shellArg(s.base)} ${s.id}`
+
+/** A stuck document, in a line with its fix. */
+export function stuckLine(s: IngestSource): string {
+  return `Knowledge base ${s.base}: ${s.fileName} has been ${s.status} since ${s.updatedAt ?? 'an earlier run'} with no ingestion behind it, so an Agent searching the base can’t find it. Fix: \`${retryCommand(s)}\` (or \`frontera local dev --once --reapply\`)`
+}
+
+/** Documents named in a short list: `a.md, b.md` or `a.md, b.md and 3 more`. */
+export function documentList(sources: IngestSource[], max = 3): string {
+  const names = sources.map((s) => s.fileName)
+  return names.length > max ? `${names.slice(0, max).join(', ')} and ${names.length - max} more` : names.join(', ')
+}
+
+export interface KnowledgeSettle {
+  /** What to say, each a line (a warning starts `warning:`). */
+  lines: string[]
+  /** Documents queued again, by base and file. */
+  requeued: string[]
+  /** Documents still being read when the wait ended. */
+  unsettled: string[]
+}
+
+/**
+ * The knowledge as the tests and the snapshot should see it: any document
+ * stuck with no job behind it queued again, then a bounded wait until none is
+ * pending or processing. `restoredAt`: when the database was last restored
+ * (a row unsettled since before it came from the snapshot); `afterRestore`
+ * queues every unsettled document again, as a restore just made them.
+ */
+export async function settleKnowledge(
+  api: IngestApi,
+  workspaceId: string,
+  opts: { restoredAt?: string | null; afterRestore?: boolean; timeoutMs?: number; pollMs?: number; sleep?: (ms: number) => Promise<void>; now?: () => number } = {},
+): Promise<KnowledgeSettle> {
+  const out: KnowledgeSettle = { lines: [], requeued: [], unsettled: [] }
+  const sources = await listIngestSources(api, workspaceId)
+  if (!sources) return out
+  const now = opts.now ?? Date.now
+  const stuck = opts.afterRestore ? sources.filter((s) => UNSETTLED.has(s.status)) : stuckSources(sources, { restoredAt: opts.restoredAt, now: now() })
+  if (stuck.length) {
+    const { requeued, refused } = await requeueSources(api, stuck)
+    out.requeued = requeued.map((s) => `${s.base}/${s.fileName}`)
+    if (requeued.length) {
+      const one = requeued.length === 1
+      const why = opts.afterRestore
+        ? `the restored database holds ${one ? 'it' : 'them'} unread, and ${one ? 'its' : 'their'} reading went with the database it replaced`
+        : `${one ? 'it was' : 'they were'} stuck with no ingestion behind ${one ? 'it' : 'them'}`
+      out.lines.push(`knowledge: queued ${documentList(requeued)} to be read again: ${why}`)
+    }
+    for (const r of refused) out.lines.push(`warning: ${stuckLine(r.source)} (re-queuing it was refused: ${r.reason})`)
+  }
+  if (!sources.some((s) => UNSETTLED.has(s.status)) && !stuck.length) return out
+  const waited = await waitForIngestion(api, workspaceId, { timeoutMs: opts.timeoutMs, pollMs: opts.pollMs, sleep: opts.sleep, now })
+  out.unsettled = waited.waiting.map((s) => `${s.base}/${s.fileName}`)
+  if (waited.settled) out.lines.push(`knowledge: every document is read (waited ${Math.round(waited.ms / 100) / 10} s)`)
+  else out.lines.push(`warning: knowledge: ${documentList(waited.waiting)} ${waited.waiting.length === 1 ? 'is' : 'are'} still being read after ${Math.round(waited.ms / 1000)} s, so an Agent searching ${waited.waiting.length === 1 ? 'its base' : 'their bases'} may not find ${waited.waiting.length === 1 ? 'it' : 'them'} yet; \`frontera knowledge sources <base>\` shows each`)
+  for (const f of waited.failed.filter((s) => stuck.some((x) => x.id === s.id))) out.lines.push(`warning: Knowledge base ${f.base}: ${f.fileName} failed to read again: \`frontera knowledge sources ${shellArg(f.base)}\` shows why`)
+  return out
+}
+
+/** The documents stuck on a running runtime, each as `local status` and the status tool list it; null when it did not answer. */
+export async function stuckKnowledge(serviceUrl: string, creds: Pick<LocalCredentials, 'workspaceKey' | 'workspaceId'>, restoredAt?: string | null) {
+  const api = localIngestApi(serviceUrl, creds)
+  const sources = api ? await listIngestSources(api, creds.workspaceId) : null
+  if (!sources) return null
+  return stuckSources(sources, { restoredAt }).map((k) => ({ base: k.base, file: k.fileName, sourceId: k.id, status: k.status, since: k.updatedAt ?? null, fix: retryCommand(k), problem: stuckLine(k) }))
+}

package/src/local/local-knowledge-uploads.ts

@@ -0,0 +1,217 @@
+import { randomBytes } from 'node:crypto'
+import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'
+import { dirname, join, resolve, sep } from 'node:path'
+
+import { KNOWLEDGE_DOCUMENT } from './local-dev-references'
+import { documentPath, documentPaths, MAX_DOCUMENT_BYTES, readFolders } from './local-knowledge-documents'
+import { QUALITY_CHECKS_FILE } from './local-quality-checks'
+import { rows } from './local-sync-remote'
+
+/**
+ * Studio's knowledge base page saves a document into the project, not onto
+ * the runtime alone: the file lands in `knowledge/<folder>/` beside its
+ * `knowledge.json`, and `local dev` uploads it as it uploads a document added
+ * by hand (a running watcher at once, or the next `local dev`). A document
+ * added only on the runtime would be lost on the next reset or restore and
+ * never reach the project's files.
+ *
+ * What is refused, so a request cannot write anywhere else or anything the
+ * apply would not ingest:
+ * - a name that is a path (a separator, `..`, a leading dot) or a control character;
+ * - a kind `local dev` does not upload (`KNOWLEDGE_DOCUMENT`);
+ * - an empty file, or one larger than the platform takes (`MAX_DOCUMENT_BYTES`);
+ * - a name already in the folder, unless the request says to replace it;
+ * - a folder or file that is a symbolic link, or resolves outside `knowledge/`.
+ *
+ * Removing a document takes its file out of the folder. The apply only adds
+ * what the runtime lacks, so the server takes the runtime's copy off first
+ * (`removeRuntimeCopies`), as it does before a replaced file is written, so
+ * the next apply uploads the new content.
+ */
+
+/** The extensions `local dev` uploads into a knowledge base, as `KNOWLEDGE_DOCUMENT` matches them. */
+export const KNOWLEDGE_UPLOAD_EXTENSIONS = ['.md', '.markdown', '.txt', '.pdf', '.docx', '.html'] as const
+export const KNOWLEDGE_UPLOAD_KINDS = 'Markdown, text, PDF, Word or HTML'
+export const MAX_KNOWLEDGE_UPLOAD_BYTES = MAX_DOCUMENT_BYTES
+
+export class KnowledgeDocumentError extends Error {
+  constructor(
+    message: string,
+    readonly status: number,
+    readonly extra: Record<string, unknown> = {},
+  ) {
+    super(message)
+  }
+}
+
+const FOLDER = /^[A-Za-z0-9][\w .-]{0,127}$/
+
+/** The project folder (`knowledge/<folder>/`) whose knowledge.json names the base `baseName`; its folder name when it gives none. */
+export function knowledgeFolderOf(root: string, baseName: string): string | null {
+  const top = join(root, 'knowledge')
+  let names: string[]
+  try {
+    names = readdirSync(top).sort()
+  } catch {
+    return null
+  }
+  const wanted = baseName.trim()
+  for (const folder of names) {
+    if (!FOLDER.test(folder)) continue
+    try {
+      if (!lstatSync(join(top, folder)).isDirectory()) continue
+      const doc = JSON.parse(readFileSync(join(top, folder, 'knowledge.json'), 'utf8')) as { name?: unknown }
+      const name = typeof doc.name === 'string' && doc.name.trim() ? doc.name.trim() : folder
+      if (name === wanted) return folder
+    } catch {
+      // No knowledge.json, or not JSON: not a knowledge base `local dev` applies.
+    }
+  }
+  return null
+}
+
+/** The knowledge base's folder, checked to be a real directory inside the project's `knowledge/`. */
+function folderDir(root: string, folder: string): string {
+  if (!FOLDER.test(folder)) throw new KnowledgeDocumentError(`"${folder}" is not a knowledge base folder`, 400)
+  const top = join(root, 'knowledge')
+  const dir = join(top, folder)
+  let st
+  try {
+    st = lstatSync(dir)
+  } catch {
+    throw new KnowledgeDocumentError(`knowledge/${folder}/ is not in the project`, 404)
+  }
+  if (!st.isDirectory()) throw new KnowledgeDocumentError(`knowledge/${folder} is not a folder`, 409)
+  if (!existsSync(join(dir, 'knowledge.json'))) throw new KnowledgeDocumentError(`knowledge/${folder}/ has no knowledge.json, so local dev does not apply its documents`, 409)
+  const real = realpathSync(dir)
+  if (!real.startsWith(realpathSync(top) + sep)) throw new KnowledgeDocumentError(`knowledge/${folder}/ is outside the project's knowledge folder`, 400)
+  return dir
+}
+
+/** A document's name as the upload gives it, checked: a plain file name of a kind `local dev` uploads. */
+export function documentName(fileName: string): string {
+  const name = fileName.normalize('NFC').trim()
+  if (!name) throw new KnowledgeDocumentError('the document has no file name', 400)
+  if (name.length > 200) throw new KnowledgeDocumentError('the file name is longer than 200 characters', 400)
+  if (/[/\\]/.test(name) || name === '.' || name === '..' || name.startsWith('.')) {
+    throw new KnowledgeDocumentError(`"${name}" is not a file name: give the document's own name, with no folder and no leading dot`, 400)
+  }
+  if (/[\x00-\x1f\x7f]/.test(name)) throw new KnowledgeDocumentError('the file name has a control character in it', 400)
+  if (name === 'knowledge.json' || name === QUALITY_CHECKS_FILE || !KNOWLEDGE_DOCUMENT.test(name)) {
+    throw new KnowledgeDocumentError(`${name} is not a kind local dev uploads into a knowledge base: use ${KNOWLEDGE_UPLOAD_KINDS} (${KNOWLEDGE_UPLOAD_EXTENSIONS.join(', ')})`, 415)
+  }
+  return name
+}
+
+/** A document's file inside the folder: `rel` is its path under the knowledge base's folder (`Policies/terms.pdf`). */
+function documentFile(dir: string, rel: string): string {
+  const segments = rel.split('/')
+  if (segments.length > 16 || segments.some((s) => !s || s === '.' || s === '..' || s.startsWith('.') || s.includes('\\') || s.includes('\0'))) {
+    throw new KnowledgeDocumentError(`"${rel}" is not a document path in the knowledge base`, 400)
+  }
+  documentName(segments[segments.length - 1]!)
+  const file = resolve(dir, ...segments)
+  if (!file.startsWith(dir + sep)) throw new KnowledgeDocumentError(`"${rel}" is outside the knowledge base's folder`, 400)
+  return file
+}
+
+/** A document's name and size, checked before anything is written or taken off the runtime; returns its name. */
+export function checkUpload(fileName: string, bytes: number): string {
+  const name = documentName(fileName)
+  if (bytes === 0) throw new KnowledgeDocumentError(`${name} is empty`, 400)
+  if (bytes > MAX_KNOWLEDGE_UPLOAD_BYTES) {
+    throw new KnowledgeDocumentError(`${name} is larger than ${MAX_KNOWLEDGE_UPLOAD_BYTES / 1024 / 1024} MB, the most a knowledge base takes`, 413)
+  }
+  return name
+}
+
+/** Whether a document of that name is in the folder already (a replace takes the runtime's copy off first). */
+export function documentInFolder(root: string, folder: string, fileName: string): boolean {
+  const file = join(folderDir(root, folder), documentName(fileName))
+  try {
+    lstatSync(file)
+    return true
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Write a document into `knowledge/<folder>/`. A name already there is
+ * refused (409, `exists: true`) unless `replace`; the file appears whole, by
+ * rename from a hidden temporary file, so the watcher never uploads half of it.
+ */
+export function saveKnowledgeDocument(root: string, folder: string, fileName: string, bytes: Uint8Array, opts: { replace?: boolean } = {}): { path: string; replaced: boolean } {
+  const name = checkUpload(fileName, bytes.byteLength)
+  const dir = folderDir(root, folder)
+  const file = join(dir, name)
+  const path = `knowledge/${folder}/${name}`
+  let existing = null
+  try {
+    existing = lstatSync(file)
+  } catch {
+    // Not there: a new document.
+  }
+  if (existing && !existing.isFile()) throw new KnowledgeDocumentError(`${path} is not a plain file, so it is left as it is`, 409)
+  if (existing && !opts.replace) throw new KnowledgeDocumentError(`${path} is in the project already`, 409, { exists: true, path })
+  const tmp = join(dir, `.${randomBytes(6).toString('hex')}.upload`)
+  try {
+    writeFileSync(tmp, bytes, { flag: 'wx', mode: 0o644 })
+    renameSync(tmp, file)
+  } catch (err) {
+    try {
+      unlinkSync(tmp)
+    } catch {
+      // Never written.
+    }
+    throw new KnowledgeDocumentError(`${path} could not be written: ${(err as Error).message}`, 500)
+  }
+  return { path, replaced: !!existing }
+}
+
+/** Take a document's file out of the folder; false when it has none there. */
+export function removeKnowledgeDocument(root: string, folder: string, rel: string): { path: string; removed: boolean } {
+  const file = documentFile(folderDir(root, folder), rel)
+  const path = `knowledge/${folder}/${rel}`
+  let st
+  try {
+    st = lstatSync(file)
+  } catch {
+    return { path, removed: false }
+  }
+  if (!st.isFile()) throw new KnowledgeDocumentError(`${path} is not a plain file, so it is left as it is`, 409)
+  // A subfolder that is a link elsewhere is not the knowledge base's.
+  const real = realpathSync(dirname(file))
+  const top = realpathSync(join(root, 'knowledge', folder))
+  if (real !== top && !real.startsWith(top + sep)) throw new KnowledgeDocumentError(`${path} is outside the knowledge base's folder`, 400)
+  unlinkSync(file)
+  return { path, removed: true }
+}
+
+/** A call to the local service as the seeded owner, returning the parsed body (`data` envelope and all). */
+export type ServiceCall = (method: 'GET' | 'DELETE', path: string) => Promise<unknown>
+
+interface RuntimeSource {
+  id: string
+  /** Its path under the knowledge base's folder, as the project files it. */
+  path: string
+}
+
+/** The runtime's documents in a knowledge base, each by its path under the folder (`Policies/terms.pdf`). */
+export async function runtimeDocuments(call: ServiceCall, knowledgeId: string): Promise<RuntimeSource[]> {
+  const listed = rows<{ id?: string; fileName?: string | null; folderId?: string | null }>(await call('GET', `/v1/workspace-knowledge/${encodeURIComponent(knowledgeId)}/sources`))
+  const folders = listed.some((s) => s.folderId) ? await readFolders({ get: (p) => call('GET', p) }, knowledgeId) : new Map<string, string>()
+  return listed.flatMap((s) => (s.id && s.fileName ? [{ id: s.id, path: documentPath(s.folderId ? (folders.get(s.folderId) ?? null) : null, s.fileName) }] : []))
+}
+
+/** Take the runtime's copies of a document off (by its path), so the next apply uploads the project's file again. */
+export async function removeRuntimeCopies(call: ServiceCall, knowledgeId: string, rel: string): Promise<number> {
+  const copies = (await runtimeDocuments(call, knowledgeId)).filter((s) => s.path === rel)
+  for (const copy of copies) await call('DELETE', `/v1/workspace-knowledge/${encodeURIComponent(knowledgeId)}/sources/${encodeURIComponent(copy.id)}`)
+  return copies.length
+}
+
+/** The project's documents for a knowledge base, each with whether the runtime holds it yet. */
+export function projectDocuments(root: string, folder: string, onRuntime: ReadonlySet<string> | null): Array<{ path: string; onRuntime: boolean | null }> {
+  return documentPaths(root, folder).map((path) => ({ path, onRuntime: onRuntime ? onRuntime.has(path) : null }))
+}

package/src/local/local-lessons.ts

@@ -96,7 +96,7 @@ export async function pullLessons(root: string, workspace: string, ws: SyncClien
   const byAgent = new Map(rows<LessonsRow>(found).map((r) => [r.agent, r.rules]))
   const record = readRecord(root, workspace)
   const out: LessonsPulled = { written: [], removed: [], kept: [], rules: 0 }
-  for (const slug of [...new Set(agents)].sort()) {
+  for (const slug of [...new Set(agents)].sort((a, b) => (String(a) < String(b) ? -1 : String(a) > String(b) ? 1 : 0))) {
     const path = join(root, lessonsPath(slug))
     const here = existsSync(path) ? readFileSync(path, 'utf8') : null
     const editedHere = here !== null && record.files[slug] !== sha(here)

package/src/local/local-library.ts

@@ -64,7 +64,7 @@ export function readLibrary(rows: LibraryRow[], directory: Directory): RemoteIte
       release: { number: r.release.number, version: r.release.version },
       ...(r.state && r.state !== 'installed' ? { state: r.state } : {}),
       ...(agent ? { agent: `agent/${agent}` } : {}),
-      ...(made.length ? { skills: [...new Set(made)].sort() } : {}),
+      ...(made.length ? { skills: [...new Set(made)].sort((a, b) => (String(a) < String(b) ? -1 : String(a) > String(b) ? 1 : 0)) } : {}),
     }
   })
   const doc: LibraryFile = { installations }

package/src/local/local-media-placeholders.ts

@@ -0,0 +1,182 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { deflateSync } from 'node:zlib'
+
+import { parse as parseYaml } from 'yaml'
+
+import { readCsvRows } from './local-sample-data'
+
+/**
+ * Placeholder files for the stand-in Media Sets (`local-media-sets.ts`): a
+ * media-backed object type has no rows locally, because the platform's files
+ * never come down. So that its catalog rows, its links and a read of a file
+ * (`blueprint_read_media`) can be tried here, `local dev` puts a few
+ * generated files in an empty stand-in set. They are placeholders and say so:
+ * a grey PNG whose text chunk names it one, or a text file that says it is
+ * one. They are made only while the set is empty, so a file someone uploaded
+ * to the local set is never mixed with them.
+ *
+ * Their paths follow the records that point at them: a link joining the
+ * type's `path` to another type's property (`Amenity.imageUrl`) takes that
+ * property's values from the other type's sample rows, so the link resolves.
+ * Without one, `placeholders/<type>-<n>.png`.
+ */
+
+export const PLACEHOLDER_COUNT = 12
+const TEXT_EXTENSIONS = /\.(txt|md|csv|json|html?|xml|ya?ml|log)$/i
+
+interface LinkFile {
+  from?: { objectType?: string; property?: string }
+  to?: { objectType?: string; property?: string }
+}
+
+function readYamlDir<T>(dir: string): T[] {
+  if (!existsSync(dir)) return []
+  return readdirSync(dir).filter((f) => /\.(ya?ml|json)$/.test(f)).flatMap((f) => {
+    try {
+      const doc = parseYaml(readFileSync(join(dir, f), 'utf8')) as T | null
+      return doc ? [doc] : []
+    } catch {
+      return []
+    }
+  })
+}
+
+/** The column a type's property is in, and the dataset it reads: its backing, else snake case. */
+function columnOf(type: { backing?: { dataset?: string; mapping?: Record<string, unknown> } }, property: string): { dataset: string | undefined; column: string } {
+  const m = type.backing?.mapping?.[property]
+  const column = typeof m === 'string' ? m : m && typeof m === 'object' && typeof (m as { column?: unknown }).column === 'string' ? (m as { column: string }).column : property.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`)
+  return { dataset: type.backing?.dataset, column }
+}
+
+/** The paths the placeholders of a media-backed type take, from the records that point at its files. */
+export function placeholderPaths(root: string, mediaType: string, max = PLACEHOLDER_COUNT): string[] {
+  const types = new Map(readYamlDir<{ apiName?: string; backing?: { dataset?: string; mapping?: Record<string, unknown> } }>(join(root, 'blueprint', 'object-types')).flatMap((t) => (t.apiName ? [[t.apiName, t] as const] : [])))
+  const media = types.get(mediaType)
+  const isPath = (property: string | undefined) => !!property && (property === 'path' || (media ? columnOf(media, property).column === 'path' : false))
+  const found = new Set<string>()
+  for (const link of readYamlDir<LinkFile>(join(root, 'blueprint', 'link-types'))) {
+    const [mine, other] = link.from?.objectType === mediaType ? [link.from, link.to] : link.to?.objectType === mediaType ? [link.to, link.from] : [null, null]
+    if (!mine || !other?.objectType || !other.property || !isPath(mine.property)) continue
+    const otherType = types.get(other.objectType)
+    if (!otherType) continue
+    const { dataset, column } = columnOf(otherType, other.property)
+    const csv = join(root, 'datasets', `${dataset ?? other.objectType.toLowerCase()}.csv`)
+    if (!existsSync(csv)) continue
+    for (const row of readCsvRows(readFileSync(csv, 'utf8'))) {
+      const value = (row[column] ?? '').trim().replace(/^\/+/, '')
+      // A media path is a path in the set: an address elsewhere (https://…) is not one of its files.
+      if (value && !/^[a-z]+:\/\//i.test(value) && !/\.\.(\/|$)/.test(value) && value.length <= 512) found.add(value)
+      if (found.size >= max) break
+    }
+  }
+  if (found.size) return [...found]
+  return defaultPlaceholderPaths(mediaType, max)
+}
+
+/**
+ * The paths a media-backed type's placeholders take when no record names one:
+ * `placeholders/<type>-<n>.png`. Sample data gives the records that point at
+ * the type's files these paths (`generateSyntheticData`), so their links
+ * resolve against the placeholders `local dev` uploads.
+ */
+export function defaultPlaceholderPaths(mediaType: string, max = PLACEHOLDER_COUNT): string[] {
+  const stem = mediaType.replace(/([a-z0-9])([A-Z])/g, '$1-$2').toLowerCase()
+  return Array.from({ length: max }, (_, i) => `placeholders/${stem}-${String(i + 1).padStart(2, '0')}.png`)
+}
+
+/** A placeholder file for a path: a text file for a text path, else a small grey PNG; each says it is a placeholder. */
+export function placeholderFile(path: string): { bytes: Uint8Array; mime: string; filename: string } {
+  const filename = path.split('/').pop() || 'placeholder'
+  const words = `Placeholder for ${path}, made by \`frontera local dev\` in the local runtime's stand-in Media Set: not the platform's file.`
+  if (TEXT_EXTENSIONS.test(path)) return { bytes: new TextEncoder().encode(`${words}\n`), mime: 'text/plain', filename }
+  return { bytes: placeholderPng(words), mime: 'image/png', filename: /\.png$/i.test(filename) ? filename : `${filename}.png` }
+}
+
+const CRC_TABLE = (() => {
+  const table = new Uint32Array(256)
+  for (let n = 0; n < 256; n++) {
+    let c = n
+    for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1
+    table[n] = c >>> 0
+  }
+  return table
+})()
+
+function crc32(bytes: Uint8Array): number {
+  let c = 0xffffffff
+  for (const b of bytes) c = CRC_TABLE[(c ^ b) & 0xff]! ^ (c >>> 8)
+  return (c ^ 0xffffffff) >>> 0
+}
+
+function chunk(type: string, data: Uint8Array): Uint8Array {
+  const out = new Uint8Array(12 + data.length)
+  const view = new DataView(out.buffer)
+  view.setUint32(0, data.length)
+  out.set(new TextEncoder().encode(type), 4)
+  out.set(data, 8)
+  view.setUint32(8 + data.length, crc32(out.subarray(4, 8 + data.length)))
+  return out
+}
+
+/** A 64×48 grey PNG with a `Comment` text chunk saying it is a placeholder. */
+export function placeholderPng(comment: string, width = 64, height = 48): Uint8Array {
+  const header = new Uint8Array(13)
+  const hv = new DataView(header.buffer)
+  hv.setUint32(0, width)
+  hv.setUint32(4, height)
+  header.set([8, 0, 0, 0, 0], 8) // 8-bit greyscale, no interlace
+  const raw = new Uint8Array((width + 1) * height)
+  for (let y = 0; y < height; y++) {
+    raw[y * (width + 1)] = 0
+    // A diagonal stripe, so it reads as a placeholder rather than a blank.
+    for (let x = 0; x < width; x++) raw[y * (width + 1) + 1 + x] = (x + y) % 16 < 8 ? 0xc8 : 0xb4
+  }
+  const text = new TextEncoder().encode(`Comment\0${comment}`)
+  const parts = [new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk('IHDR', header), chunk('tEXt', text), chunk('IDAT', new Uint8Array(deflateSync(raw))), chunk('IEND', new Uint8Array())]
+  const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0))
+  let at = 0
+  for (const p of parts) {
+    out.set(p, at)
+    at += p.length
+  }
+  return out
+}
+
+type Call = (method: string, path: string, body?: unknown) => Promise<{ status: number; body: unknown }>
+type Upload = (mediaSetId: string, file: { bytes: Uint8Array; mime: string; filename: string; logicalPath: string }) => Promise<{ status: number; body: unknown }>
+
+const dataOf = (body: unknown) => ((body as { data?: unknown } | null)?.data ?? body) as Record<string, unknown> | null
+
+/**
+ * Fill each empty stand-in set with placeholders for its type: the lines
+ * `local dev` says. A set that holds any file is left alone; a refused upload
+ * is said and stops that set.
+ */
+export async function fillPlaceholderMedia(root: string, mediaTypes: Readonly<Record<string, { set: string }>>, call: Call, upload: Upload): Promise<string[]> {
+  const lines: string[] = []
+  const listed = await call('GET', '/v1/media/sets')
+  if (listed.status >= 400) return [`  warning: the stand-in Media Sets couldn’t be read for placeholder files (${listed.status})`]
+  const ids = new Map(((dataOf(listed.body)?.mediaSets as Array<{ id?: string; apiName?: string }> | undefined) ?? []).map((s) => [s.apiName, s.id] as const))
+  const done = new Set<string>()
+  for (const [type, { set }] of Object.entries(mediaTypes)) {
+    const id = ids.get(set)
+    if (!id || done.has(set)) continue
+    done.add(set)
+    const items = await call('GET', `/v1/media/sets/${encodeURIComponent(id)}/items`)
+    if (items.status >= 400 || ((dataOf(items.body)?.items as unknown[] | undefined) ?? []).length) continue
+    const paths = placeholderPaths(root, type)
+    let made = 0
+    for (const logicalPath of paths) {
+      const answer = await upload(id, { ...placeholderFile(logicalPath), logicalPath })
+      if (answer.status >= 400) {
+        const message = (answer.body as { message?: unknown } | null)?.message
+        lines.push(`  warning: ${set}: a placeholder file couldn’t be added (${answer.status}${typeof message === 'string' ? ` ${message.slice(0, 160)}` : ''}), so ${type} has no rows here`)
+        break
+      }
+      made++
+    }
+    if (made) lines.push(`  ${type}: ${made} placeholder file${made === 1 ? '' : 's'} in the stand-in Media Set ${set} (generated here, not the platform’s files), so its rows and links can be tried`)
+  }
+  return lines
+}