@frontera-sdk/cli 1.52.42 → 1.52.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/commands/local/check.ts +17 -9
- package/src/commands/local/index-commands.ts +7 -1
- package/src/commands/local/snapshot.ts +8 -2
- package/src/commands/local/sync-request.ts +95 -9
- package/src/commands/local/sync.ts +27 -6
- package/src/commands/registry.ts +2 -1
- package/src/local/local-case-evidence.ts +3 -2
- package/src/local/local-db-snapshot.ts +23 -3
- package/src/local/local-dev-references.ts +59 -7
- package/src/local/local-env.ts +21 -0
- package/src/local/local-history-carry.ts +103 -0
- package/src/local/local-key-custodian.ts +64 -0
- package/src/local/local-knowledge-uploads.ts +62 -5
- package/src/local/local-scaffold.ts +48 -13
- package/src/local/local-schemas.ts +1 -1
- package/src/local/local-studio-server.ts +1 -1
- package/src/local/local-supervisor.ts +13 -0
- package/src/local/local-sync-link.ts +43 -2
- package/src/local/local-sync-models.ts +118 -0
- package/src/local/local-sync-plan.ts +19 -9
- package/src/local/local-sync-portable.ts +269 -0
- package/src/local/local-sync-precheck.ts +94 -0
- package/src/local/local-sync-push.ts +45 -7
- package/src/local/local-sync-remote.ts +39 -8
- package/src/local/local-sync-snapshot.ts +9 -1
|
@@ -875,7 +875,8 @@ async function localModels(local: { serviceUrl: string; creds: LocalCredentials
|
|
|
875
875
|
* A knowledge base: created when the runtime has none by its name, given the
|
|
876
876
|
* file's settings (retrieval mode, chunking, top-K and threshold, hybrid
|
|
877
877
|
* search and reranking, extraction and OCR), then the folder's documents the
|
|
878
|
-
* base does not hold yet are uploaded
|
|
878
|
+
* base does not hold yet are uploaded, and those edited here since are
|
|
879
|
+
* uploaded again in place of the runtime's older copy.
|
|
879
880
|
*/
|
|
880
881
|
/**
|
|
881
882
|
* What to say of a knowledge base with no documents in its folder: an Agent
|
|
@@ -987,7 +988,7 @@ export async function applyKnowledge(
|
|
|
987
988
|
return await done({ ok: true, output: '' })
|
|
988
989
|
}
|
|
989
990
|
const sources = await run(['knowledge', 'sources', name, '--json', ...p])
|
|
990
|
-
const listedSources = parseJson<Array<{ id?: string; fileName?: string; folderId?: string | null; status?: string; updatedAt?: string }>>(sources.output) ?? []
|
|
991
|
+
const listedSources = parseJson<Array<{ id?: string; fileName?: string; folderId?: string | null; status?: string; updatedAt?: string; sizeBytes?: number | null }>>(sources.output) ?? []
|
|
991
992
|
const key = local?.creds.workspaceKey
|
|
992
993
|
const http = async (method: string, path: string, body?: FormData | Record<string, unknown>) => {
|
|
993
994
|
const res = await fetch(new URL(path, local!.serviceUrl), {
|
|
@@ -1008,15 +1009,54 @@ export async function applyKnowledge(
|
|
|
1008
1009
|
// A document the runtime holds pending or processing with no ingestion behind it (a rollback took its
|
|
1009
1010
|
// reading with the database it replaced) is queued to be read again: uploading it again would add a second.
|
|
1010
1011
|
if (key && id) notes.push(...(await requeueStuckDocuments(project, local!, name, id, listedSources)))
|
|
1011
|
-
|
|
1012
|
-
|
|
1012
|
+
// A document edited in place keeps its name: the runtime's older copy is found by content, uploaded over, then taken off.
|
|
1013
|
+
const { loadUploadedDocuments, saveUploadedDocuments, staleDocuments } = await import('./local-knowledge-uploads')
|
|
1014
|
+
const { sha256Of } = await import('./local-knowledge-documents')
|
|
1015
|
+
const contents = documents.flatMap((path) => {
|
|
1016
|
+
try {
|
|
1017
|
+
const bytes = readFileSync(join(project.root, dirname(target.file), path))
|
|
1018
|
+
return [{ path, sha256: sha256Of(bytes), bytes: bytes.byteLength }]
|
|
1019
|
+
} catch {
|
|
1020
|
+
return []
|
|
1021
|
+
}
|
|
1022
|
+
})
|
|
1023
|
+
const onRuntime = listedSources.flatMap((s) => (s.id ? [{ id: s.id, path: documentPath(s.folderId ? folders.get(s.folderId) : null, s.fileName ?? ''), sizeBytes: s.sizeBytes }] : []))
|
|
1024
|
+
const uploaded = key && id ? loadUploadedDocuments(project.stateDir, local!.creds.organizationId, id) : {}
|
|
1025
|
+
const stale = staleDocuments(contents, onRuntime, uploaded)
|
|
1026
|
+
if (stale.length && (!key || !id)) {
|
|
1027
|
+
notes.push(` warning: Knowledge base ${name}: ${stale.map((s) => s.path).join(', ')} changed here, but the runtime keeps ${stale.length === 1 ? 'its older copy' : 'their older copies'} until it has a workspace key`)
|
|
1028
|
+
}
|
|
1029
|
+
const replacing = key && id ? stale : []
|
|
1030
|
+
// What the runtime now holds for each document, recorded so the next apply tells an edit from no change.
|
|
1031
|
+
const record = async () => {
|
|
1032
|
+
if (!key || !id) return
|
|
1033
|
+
const now = rows<{ id?: string; fileName?: string; folderId?: string | null; createdAt?: string }>(await http('GET', `/v1/workspace-knowledge/${encodeURIComponent(id)}/sources`).catch(() => null))
|
|
1034
|
+
// Folders this apply made are not in `folders` yet.
|
|
1035
|
+
const filedNow = now.some((s) => s.folderId && !folders.has(s.folderId))
|
|
1036
|
+
? folderPaths((((await http('GET', `/v1/workspace-knowledge/${encodeURIComponent(id)}/folders`).catch(() => null))?.folders ?? []) as Array<{ id: string; parentId?: string | null; name: string }>))
|
|
1037
|
+
: folders
|
|
1038
|
+
const next: typeof uploaded = {}
|
|
1039
|
+
for (const doc of contents) {
|
|
1040
|
+
const copies = now.filter((s) => s.id && documentPath(s.folderId ? filedNow.get(s.folderId) : null, s.fileName ?? '') === doc.path)
|
|
1041
|
+
const newest = copies.sort((a, b) => String(b.createdAt ?? '').localeCompare(String(a.createdAt ?? '')))[0]
|
|
1042
|
+
if (newest?.id) next[doc.path] = { sourceId: newest.id, sha256: doc.sha256 }
|
|
1043
|
+
}
|
|
1044
|
+
saveUploadedDocuments(project.stateDir, local!.creds.organizationId, id, next)
|
|
1045
|
+
}
|
|
1046
|
+
const fresh = [...documents.filter((f) => !have.has(f)), ...replacing.map((s) => s.path)]
|
|
1047
|
+
if (!fresh.length) {
|
|
1048
|
+
await record()
|
|
1049
|
+
return await done({ ok: true, output: '' })
|
|
1050
|
+
}
|
|
1013
1051
|
const top = fresh.filter((f) => !f.includes('/'))
|
|
1014
1052
|
const filed = fresh.filter((f) => f.includes('/'))
|
|
1015
1053
|
const lines: string[] = []
|
|
1054
|
+
const sent = new Set<string>()
|
|
1016
1055
|
if (top.length) {
|
|
1017
|
-
const
|
|
1018
|
-
if (!
|
|
1019
|
-
if (
|
|
1056
|
+
const uploadedTop = await run(['knowledge', 'upload', name, ...top.map((f) => join(dirname(target.file), f)), ...p])
|
|
1057
|
+
if (!uploadedTop.ok) return await done(uploadedTop)
|
|
1058
|
+
if (uploadedTop.output.trim()) lines.push(uploadedTop.output.trim())
|
|
1059
|
+
for (const f of top) sent.add(f)
|
|
1020
1060
|
}
|
|
1021
1061
|
if (filed.length && (!key || !id)) {
|
|
1022
1062
|
notes.push(` warning: Knowledge base ${name}: ${filed.length} document${filed.length === 1 ? ' in a folder waits' : 's in folders wait'} until the runtime has a workspace key`)
|
|
@@ -1048,11 +1088,23 @@ export async function applyKnowledge(
|
|
|
1048
1088
|
form.append('folderId', await folderFor(folder!))
|
|
1049
1089
|
await http('POST', `/v1/workspace-knowledge/${encodeURIComponent(id!)}/sources`, form)
|
|
1050
1090
|
lines.push(` uploaded ${path}`)
|
|
1091
|
+
sent.add(path)
|
|
1051
1092
|
} catch (err) {
|
|
1052
1093
|
notes.push(` warning: Knowledge base ${name}: ${path} wasn’t uploaded (${(err as Error).message})`)
|
|
1053
1094
|
}
|
|
1054
1095
|
}
|
|
1055
1096
|
}
|
|
1097
|
+
// The older copies come off only once the new content is up, so the base is never without the document.
|
|
1098
|
+
for (const doc of replacing) {
|
|
1099
|
+
if (!sent.has(doc.path)) continue
|
|
1100
|
+
try {
|
|
1101
|
+
for (const sourceId of doc.replaces) await http('DELETE', `/v1/workspace-knowledge/${encodeURIComponent(id!)}/sources/${encodeURIComponent(sourceId)}`)
|
|
1102
|
+
lines.push(` replaced ${doc.path}: it changed here`)
|
|
1103
|
+
} catch (err) {
|
|
1104
|
+
notes.push(` warning: Knowledge base ${name}: ${doc.path}’s older copy wasn’t taken off (${(err as Error).message}); search may find both`)
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
await record()
|
|
1056
1108
|
return await done({ ok: true, output: lines.join('\n') })
|
|
1057
1109
|
}
|
|
1058
1110
|
|
package/src/local/local-env.ts
CHANGED
|
@@ -118,6 +118,21 @@ export const SERVICE_OUTBOUND = [
|
|
|
118
118
|
'AUTOMATION_RUNNER_URL',
|
|
119
119
|
] as const
|
|
120
120
|
|
|
121
|
+
/** The service's settings for a hosted key custodian (OpenBao / Vault transit), blanked on the local runtime. */
|
|
122
|
+
export const KEY_CUSTODIAN_SETTINGS = [
|
|
123
|
+
'BAO_ADDR',
|
|
124
|
+
'BAO_TOKEN',
|
|
125
|
+
'BAO_CACERT',
|
|
126
|
+
'VAULT_ADDR',
|
|
127
|
+
'VAULT_TOKEN',
|
|
128
|
+
'SECRETS_TRANSIT_AUTH',
|
|
129
|
+
'SECRETS_TRANSIT_MOUNT',
|
|
130
|
+
'SECRETS_TRANSIT_KEY',
|
|
131
|
+
'SECRETS_TRANSIT_K8S_ROLE',
|
|
132
|
+
'SECRETS_TRANSIT_K8S_MOUNT',
|
|
133
|
+
'SECRETS_TRANSIT_K8S_TOKEN_FILE',
|
|
134
|
+
] as const
|
|
135
|
+
|
|
121
136
|
/** What the Function runner dials on this machine. */
|
|
122
137
|
export const RUNNER_OUTBOUND = ['SERVICE_URL', 'S3_ENDPOINT', 'INNGEST_BASE_URL'] as const
|
|
123
138
|
|
|
@@ -182,6 +197,12 @@ export function serviceEnv(args: {
|
|
|
182
197
|
DAYTONA_API_KEY: '',
|
|
183
198
|
DAYTONA_API_URL: '',
|
|
184
199
|
DAYTONA_TARGET: '',
|
|
200
|
+
// The runtime holds its own keys: ENCRYPTION_KEY from its secrets.json wraps every data key.
|
|
201
|
+
// A checkout's .env can name a hosted OpenBao instead (the service loads it from its
|
|
202
|
+
// directory), and every data key wrapped there is unreadable once that OpenBao goes away.
|
|
203
|
+
// Empty values win over the file.
|
|
204
|
+
SECRETS_KEY_PROVIDER: 'env',
|
|
205
|
+
...Object.fromEntries(KEY_CUSTODIAN_SETTINGS.map((key) => [key, ''])),
|
|
185
206
|
// Traces go to the runtime's own local traces, read back by Studio.
|
|
186
207
|
LANGFUSE_BASE_URL: `http://127.0.0.1:${ports.traces}`,
|
|
187
208
|
LANGFUSE_PUBLIC_KEY: LOCAL_TRACE_KEYS.publicKey,
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
import { POSTGRES_DB, POSTGRES_USER } from './local-env'
|
|
4
|
+
import type { QueryFn } from './local-secret-carry'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The run history carried across a database restore.
|
|
8
|
+
*
|
|
9
|
+
* A restore (`local check`'s rollback, `frontera local snapshot restore`)
|
|
10
|
+
* throws away what was written since the baseline. What happened is not test
|
|
11
|
+
* data: the conversations (an Agent test case's included), Agent sessions and
|
|
12
|
+
* runs, workflow and Function runs and evaluation runs are how a change is
|
|
13
|
+
* compared with the one before it. So a restore keeps them as they are now,
|
|
14
|
+
* unless asked for the baseline's (`--exact`). The records those runs wrote
|
|
15
|
+
* are still rolled back.
|
|
16
|
+
*
|
|
17
|
+
* The rows are dumped inside the database container before the database is
|
|
18
|
+
* dropped (`pg_dump --data-only --inserts --on-conflict-do-nothing`), then
|
|
19
|
+
* loaded into the restored one, a row a statement: a row the baseline holds
|
|
20
|
+
* stays as the baseline holds it, and a row that names something the restore
|
|
21
|
+
* took away (a work item a test made) is left out and counted. Nothing leaves
|
|
22
|
+
* the container, and the dump is removed once loaded.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** The tables that hold what happened, parents before children (pg_dump orders them by their keys too). */
|
|
26
|
+
export const HISTORY_TABLES = [
|
|
27
|
+
'conversations',
|
|
28
|
+
'messages',
|
|
29
|
+
'message_feedback',
|
|
30
|
+
'conversation_labels',
|
|
31
|
+
'conversation_files',
|
|
32
|
+
'trace_annotations',
|
|
33
|
+
'session_handoffs',
|
|
34
|
+
'agent_sessions',
|
|
35
|
+
'agent_session_events',
|
|
36
|
+
'agent_session_analyses',
|
|
37
|
+
'agent_timeline_events',
|
|
38
|
+
'agent_exchanges',
|
|
39
|
+
'pi_runs',
|
|
40
|
+
'pi_run_events',
|
|
41
|
+
'pi_model_calls',
|
|
42
|
+
'workflow_runs',
|
|
43
|
+
'workflow_run_nodes',
|
|
44
|
+
'workflow_step_test_results',
|
|
45
|
+
'function_runs',
|
|
46
|
+
'function_run_steps',
|
|
47
|
+
'ctx_agent_turns',
|
|
48
|
+
'dataset_runs',
|
|
49
|
+
'dataset_run_items',
|
|
50
|
+
'knowledge_eval_runs',
|
|
51
|
+
'knowledge_eval_run_fixtures',
|
|
52
|
+
'scheduled_task_runs',
|
|
53
|
+
'agent_routine_runs',
|
|
54
|
+
] as const
|
|
55
|
+
|
|
56
|
+
/** The history dumped from the current database, waiting in the container for the restored one. */
|
|
57
|
+
export interface HistoryHeld {
|
|
58
|
+
file: string
|
|
59
|
+
rows: number
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface HistoryCarried {
|
|
63
|
+
ok: boolean
|
|
64
|
+
/** Rows dumped before the restore (those the baseline already held included). */
|
|
65
|
+
rows: number
|
|
66
|
+
/** Rows left out: they name something the restored database lacks. */
|
|
67
|
+
notKept: number
|
|
68
|
+
/** Why nothing was carried, when it failed. */
|
|
69
|
+
problem?: string
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const sh = (script: string) => ['sh', '-c', script]
|
|
73
|
+
|
|
74
|
+
/** Dump the history tables as they are now into a file in the container, or why it could not be. */
|
|
75
|
+
export function dumpHistory(query: QueryFn, container: string): HistoryHeld | { problem: string } {
|
|
76
|
+
const file = `/tmp/frontera-history-${randomBytes(6).toString('hex')}.sql`
|
|
77
|
+
const tables = HISTORY_TABLES.map((t) => `-t public.${t}`).join(' ')
|
|
78
|
+
const res = query(container, sh(`pg_dump -U ${POSTGRES_USER} -d ${POSTGRES_DB} --data-only --inserts --on-conflict-do-nothing ${tables} -f ${file} || exit 1; grep -c '^INSERT INTO' ${file}; exit 0`))
|
|
79
|
+
const rows = Number.parseInt(res.out.trim().split('\n').pop() ?? '', 10)
|
|
80
|
+
if (!res.ok || !Number.isFinite(rows)) {
|
|
81
|
+
query(container, sh(`rm -f ${file}`))
|
|
82
|
+
return { problem: 'the run history could not be read' }
|
|
83
|
+
}
|
|
84
|
+
return { file, rows }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Load the dumped history into the restored database, then remove the dump. */
|
|
88
|
+
export function loadHistory(query: QueryFn, container: string, held: HistoryHeld): HistoryCarried {
|
|
89
|
+
// No ON_ERROR_STOP: each row is its own statement, and one the restored database refuses is counted, not fatal.
|
|
90
|
+
const res = query(container, sh(`psql -U ${POSTGRES_USER} -d ${POSTGRES_DB} -q -f ${held.file} 2>&1 | grep -c 'ERROR:'; rm -f ${held.file}`))
|
|
91
|
+
const notKept = Number.parseInt(res.out.trim().split('\n').pop() ?? '', 10)
|
|
92
|
+
if (!res.ok || !Number.isFinite(notKept)) return { ok: false, rows: held.rows, notKept: 0, problem: 'the run history could not be written back' }
|
|
93
|
+
return { ok: true, rows: held.rows, notKept }
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** What a restore does with the run history, in words. */
|
|
97
|
+
export function historyWords(history: (HistoryCarried & { mode: 'keep' }) | { mode: 'baseline' } | undefined): string {
|
|
98
|
+
if (!history) return ''
|
|
99
|
+
if (history.mode === 'baseline') return 'the run history is as the baseline holds it (--exact)'
|
|
100
|
+
if (!history.ok) return `the run history couldn’t be kept (${history.problem ?? 'unknown'}), so conversations and runs are as the baseline holds them`
|
|
101
|
+
const kept = 'kept the run history (conversations, runs and evaluations)'
|
|
102
|
+
return history.notKept ? `${kept}, except ${history.notKept} row${history.notKept === 1 ? '' : 's'} naming records the rollback took away` : kept
|
|
103
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { withFix } from './fix-words'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The local runtime's key custodian, as its service reports it on
|
|
5
|
+
* `/v1/health`: whether the keys that lock its secrets and governed Action
|
|
6
|
+
* records answer. A runtime holds its own keys (`serviceEnv` pins the env
|
|
7
|
+
* custodian); one whose data keys were wrapped by a hosted OpenBao it no
|
|
8
|
+
* longer reaches cannot read them again, so the only fix is a clean runtime.
|
|
9
|
+
* Read by `local start` (which refuses), `local status`, `local check` and
|
|
10
|
+
* Forge's Runtime page. The report carries categories and a count, never a
|
|
11
|
+
* key, token or address.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export interface KeyCustodianReport {
|
|
15
|
+
state: 'ready' | 'failed' | 'none'
|
|
16
|
+
provider: 'env' | 'transit' | null
|
|
17
|
+
reason?: 'unreachable' | 'misconfigured' | 'unreadable-keys'
|
|
18
|
+
foreignKeys: number
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** What fixes a broken custodian: nothing else can unwrap those data keys. */
|
|
22
|
+
export const KEY_CUSTODIAN_FIX = 'run `frontera local reset` for a clean runtime that holds its own keys (it wipes this runtime’s data)'
|
|
23
|
+
|
|
24
|
+
/** How every custodian problem starts, so a failed start's error is recognised as one. */
|
|
25
|
+
export const KEY_CUSTODIAN_LEAD = 'the runtime’s encryption keys can’t be read'
|
|
26
|
+
|
|
27
|
+
/** The problem a report describes, with its fix; null when the keys answer (or the report is from a service that has none). */
|
|
28
|
+
export function keyCustodianProblem(report: KeyCustodianReport | null | undefined): string | null {
|
|
29
|
+
if (!report) return null
|
|
30
|
+
const what = 'so its secrets and governed Action records can’t be read or saved'
|
|
31
|
+
if (report.foreignKeys > 0) {
|
|
32
|
+
const keys = report.foreignKeys === 1 ? '1 data key was' : `${report.foreignKeys} data keys were`
|
|
33
|
+
return withFix(`${KEY_CUSTODIAN_LEAD}: ${keys} locked by a key service it doesn’t hold (an OpenBao named in a checkout’s .env), ${what}`, KEY_CUSTODIAN_FIX)
|
|
34
|
+
}
|
|
35
|
+
if (report.state !== 'failed') return null
|
|
36
|
+
const why =
|
|
37
|
+
report.reason === 'unreachable'
|
|
38
|
+
? 'its key service doesn’t answer'
|
|
39
|
+
: report.reason === 'unreadable-keys'
|
|
40
|
+
? 'its stored data keys don’t open with the keys in .frontera/local/secrets.json'
|
|
41
|
+
: 'its key service is set up wrong'
|
|
42
|
+
return withFix(`${KEY_CUSTODIAN_LEAD}: ${why}, ${what}`, KEY_CUSTODIAN_FIX)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Whether a failed start's error is a custodian problem (its fix is a reset, not a restart). */
|
|
46
|
+
export function isKeyCustodianProblem(error: string | null | undefined): boolean {
|
|
47
|
+
return !!error && error.includes(KEY_CUSTODIAN_LEAD)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The custodian problem the running service reports, or null: its keys
|
|
52
|
+
* answer, it can't be reached (other checks say so), or it is a release from
|
|
53
|
+
* before the report.
|
|
54
|
+
*/
|
|
55
|
+
export async function runtimeKeyCustodianProblem(serviceUrl: string, doFetch: (url: string, init?: RequestInit) => Promise<Response> = fetch): Promise<string | null> {
|
|
56
|
+
try {
|
|
57
|
+
const res = await doFetch(`${serviceUrl.replace(/\/+$/, '')}/v1/health`, { signal: AbortSignal.timeout(3000) })
|
|
58
|
+
if (!res.ok) return null
|
|
59
|
+
const body = (await res.json().catch(() => null)) as { data?: { keyCustodian?: KeyCustodianReport | null } } | null
|
|
60
|
+
return keyCustodianProblem(body?.data?.keyCustodian)
|
|
61
|
+
} catch {
|
|
62
|
+
return null
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto'
|
|
2
|
-
import { existsSync, lstatSync, readdirSync, readFileSync, realpathSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
2
|
+
import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
3
3
|
import { dirname, join, resolve, sep } from 'node:path'
|
|
4
4
|
|
|
5
5
|
import { KNOWLEDGE_DOCUMENT } from './local-dev-references'
|
|
@@ -23,10 +23,10 @@ import { rows } from './local-sync-remote'
|
|
|
23
23
|
* - a name already in the folder, unless the request says to replace it;
|
|
24
24
|
* - a folder or file that is a symbolic link, or resolves outside `knowledge/`.
|
|
25
25
|
*
|
|
26
|
-
* Removing a document takes its file out of the folder
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
26
|
+
* Removing a document takes its file out of the folder, and the server takes
|
|
27
|
+
* the runtime's copy off (`removeRuntimeCopies`), as it does before a replaced
|
|
28
|
+
* file is written. A file edited in place (by a coding agent, say) the apply
|
|
29
|
+
* uploads again itself, then takes the older copy off (`staleDocuments`).
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
32
|
/** The extensions `local dev` uploads into a knowledge base, as `KNOWLEDGE_DOCUMENT` matches them. */
|
|
@@ -211,6 +211,63 @@ export async function removeRuntimeCopies(call: ServiceCall, knowledgeId: string
|
|
|
211
211
|
return copies.length
|
|
212
212
|
}
|
|
213
213
|
|
|
214
|
+
/** What the last apply uploaded for each document, by its path: the runtime's copy and the content it was. */
|
|
215
|
+
export type UploadedDocuments = Record<string, { sourceId: string; sha256: string }>
|
|
216
|
+
|
|
217
|
+
const uploadsPath = (stateDir: string) => join(stateDir, 'knowledge-uploads.json')
|
|
218
|
+
|
|
219
|
+
/** The documents the apply uploaded into one knowledge base on this runtime; nothing after a reset (another organization). */
|
|
220
|
+
export function loadUploadedDocuments(stateDir: string, organizationId: string, knowledgeId: string): UploadedDocuments {
|
|
221
|
+
try {
|
|
222
|
+
const saved = JSON.parse(readFileSync(uploadsPath(stateDir), 'utf8')) as { organizationId?: string; bases?: Record<string, UploadedDocuments> }
|
|
223
|
+
if (saved.organizationId === organizationId) return saved.bases?.[knowledgeId] ?? {}
|
|
224
|
+
} catch {
|
|
225
|
+
// Missing or unreadable: no copy counts as recorded.
|
|
226
|
+
}
|
|
227
|
+
return {}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export function saveUploadedDocuments(stateDir: string, organizationId: string, knowledgeId: string, uploaded: UploadedDocuments): void {
|
|
231
|
+
let bases: Record<string, UploadedDocuments> = {}
|
|
232
|
+
try {
|
|
233
|
+
const saved = JSON.parse(readFileSync(uploadsPath(stateDir), 'utf8')) as { organizationId?: string; bases?: Record<string, UploadedDocuments> }
|
|
234
|
+
if (saved.organizationId === organizationId && saved.bases) bases = saved.bases
|
|
235
|
+
} catch {
|
|
236
|
+
// Starts afresh.
|
|
237
|
+
}
|
|
238
|
+
try {
|
|
239
|
+
mkdirSync(stateDir, { recursive: true })
|
|
240
|
+
writeFileSync(uploadsPath(stateDir), `${JSON.stringify({ organizationId, bases: { ...bases, [knowledgeId]: uploaded } })}\n`, { mode: 0o600 })
|
|
241
|
+
} catch {
|
|
242
|
+
// Losing this only means the next apply judges an edit by the copy's size.
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* The documents the runtime holds an older copy of, each with the copies to
|
|
248
|
+
* take off once the new content is up. A copy the last apply recorded is
|
|
249
|
+
* older when the file's content differs from what was uploaded; a copy no
|
|
250
|
+
* apply here recorded (uploaded before the record, or by hand) is older when
|
|
251
|
+
* its size differs, and otherwise taken as the file's.
|
|
252
|
+
*/
|
|
253
|
+
export function staleDocuments(
|
|
254
|
+
local: Array<{ path: string; sha256: string; bytes: number }>,
|
|
255
|
+
runtime: Array<{ id: string; path: string; sizeBytes?: number | null }>,
|
|
256
|
+
uploaded: UploadedDocuments,
|
|
257
|
+
): Array<{ path: string; replaces: string[] }> {
|
|
258
|
+
const out: Array<{ path: string; replaces: string[] }> = []
|
|
259
|
+
for (const doc of local) {
|
|
260
|
+
const copies = runtime.filter((s) => s.path === doc.path)
|
|
261
|
+
if (!copies.length) continue
|
|
262
|
+
const was = uploaded[doc.path]
|
|
263
|
+
const stale = was && copies.some((c) => c.id === was.sourceId)
|
|
264
|
+
? was.sha256 !== doc.sha256
|
|
265
|
+
: copies.length === 1 && typeof copies[0]!.sizeBytes === 'number' && copies[0]!.sizeBytes !== doc.bytes
|
|
266
|
+
if (stale) out.push({ path: doc.path, replaces: copies.map((c) => c.id) })
|
|
267
|
+
}
|
|
268
|
+
return out
|
|
269
|
+
}
|
|
270
|
+
|
|
214
271
|
/** The project's documents for a knowledge base, each with whether the runtime holds it yet. */
|
|
215
272
|
export function projectDocuments(root: string, folder: string, onRuntime: ReadonlySet<string> | null): Array<{ path: string; onRuntime: boolean | null }> {
|
|
216
273
|
return documentPaths(root, folder).map((path) => ({ path, onRuntime: onRuntime ? onRuntime.has(path) : null }))
|
|
@@ -587,6 +587,53 @@ export function ensureProjectBunfig(root: string): boolean {
|
|
|
587
587
|
return true
|
|
588
588
|
}
|
|
589
589
|
|
|
590
|
+
/** The example Agent, workflow, Function, evaluation and object type, with the sample dataset they use. */
|
|
591
|
+
function sampleFiles(name: string, agentModel?: string): Record<string, string> {
|
|
592
|
+
return {
|
|
593
|
+
'agents/assistant/agent.json': AGENT(name, agentModel),
|
|
594
|
+
'agents/assistant/operating.json': OPERATING,
|
|
595
|
+
'agents/assistant/bindings.json': BINDINGS,
|
|
596
|
+
'agents/assistant/evals/regression.yaml': EVALS,
|
|
597
|
+
'workflows/hello.yaml': WORKFLOW,
|
|
598
|
+
'workflows/tests/hello.yaml': WORKFLOW_TESTS,
|
|
599
|
+
'functions/word-count.ts': FUNCTION,
|
|
600
|
+
'functions/word-count.test.ts': FUNCTION_TEST,
|
|
601
|
+
'datasets/tickets.json': DATASET,
|
|
602
|
+
'datasets/tickets.csv': DATASET_ROWS,
|
|
603
|
+
'blueprint/object-types/Ticket.yaml': OBJECT_TYPE,
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/** The sample Agent as compared: its model and the project's name in its description are set per project. */
|
|
608
|
+
function sampleAgent(text: string): unknown {
|
|
609
|
+
try {
|
|
610
|
+
const doc = JSON.parse(text) as { config?: Record<string, unknown>; modelConfigs?: unknown }
|
|
611
|
+
const { description: _description, defaultMode: _defaultMode, ...config } = doc.config ?? {}
|
|
612
|
+
const { modelConfigs: _models, ...rest } = doc
|
|
613
|
+
return { ...rest, config }
|
|
614
|
+
} catch {
|
|
615
|
+
return null
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
/**
|
|
620
|
+
* Whether an item's files are still exactly what setting up the project wrote
|
|
621
|
+
* for its sample: the example, not the project's own work, so a linked
|
|
622
|
+
* project does not propose it by default: the sample Ticket type would
|
|
623
|
+
* otherwise overwrite a workspace's own. Edited, it is the project's.
|
|
624
|
+
*/
|
|
625
|
+
export function isScaffoldSample(files: Record<string, string>): boolean {
|
|
626
|
+
const sample = sampleFiles('')
|
|
627
|
+
const paths = Object.keys(files)
|
|
628
|
+
if (!paths.length) return false
|
|
629
|
+
return paths.every((path) => {
|
|
630
|
+
const expected = sample[path]
|
|
631
|
+
if (expected === undefined) return false
|
|
632
|
+
if (path === 'agents/assistant/agent.json') return JSON.stringify(sampleAgent(files[path]!)) === JSON.stringify(sampleAgent(expected))
|
|
633
|
+
return files[path] === expected
|
|
634
|
+
})
|
|
635
|
+
}
|
|
636
|
+
|
|
590
637
|
/** `samples: false` leaves out the example Agent, workflow, Function and object type: a project started from a workspace gets the workspace's own. */
|
|
591
638
|
export function solutionScaffoldFiles(name: string, opts: { agentModel?: string; samples?: boolean } = {}): Record<string, string> {
|
|
592
639
|
const sdk = vendored.files as Record<string, string>
|
|
@@ -610,19 +657,7 @@ export function solutionScaffoldFiles(name: string, opts: { agentModel?: string;
|
|
|
610
657
|
out['.codex/config.toml'] = CODEX_MCP_TOML
|
|
611
658
|
out['fixtures/README.md'] = FIXTURES_README
|
|
612
659
|
out['bunfig.toml'] = PROJECT_BUNFIG
|
|
613
|
-
if (opts.samples !== false)
|
|
614
|
-
out['agents/assistant/agent.json'] = AGENT(name, opts.agentModel)
|
|
615
|
-
out['agents/assistant/operating.json'] = OPERATING
|
|
616
|
-
out['agents/assistant/bindings.json'] = BINDINGS
|
|
617
|
-
out['agents/assistant/evals/regression.yaml'] = EVALS
|
|
618
|
-
out['workflows/hello.yaml'] = WORKFLOW
|
|
619
|
-
out['workflows/tests/hello.yaml'] = WORKFLOW_TESTS
|
|
620
|
-
out['functions/word-count.ts'] = FUNCTION
|
|
621
|
-
out['functions/word-count.test.ts'] = FUNCTION_TEST
|
|
622
|
-
out['datasets/tickets.json'] = DATASET
|
|
623
|
-
out['datasets/tickets.csv'] = DATASET_ROWS
|
|
624
|
-
out['blueprint/object-types/Ticket.yaml'] = OBJECT_TYPE
|
|
625
|
-
}
|
|
660
|
+
if (opts.samples !== false) Object.assign(out, sampleFiles(name, opts.agentModel))
|
|
626
661
|
out['package.json'] =
|
|
627
662
|
JSON.stringify(
|
|
628
663
|
{
|
|
@@ -134,7 +134,7 @@ const EVALS_SCHEMA: JsonSchema = {
|
|
|
134
134
|
conversation: {
|
|
135
135
|
type: 'array',
|
|
136
136
|
minItems: 1,
|
|
137
|
-
description: 'One turn per item. Only the user turns are sent, in order, each as soon as the reply before it ends; an assistant turn is kept for whoever reads the case. `waitSeconds` on a user turn keeps a gap before it (`- { user: "Still there?", waitSeconds: 60 }` sends it a minute after the reply before).',
|
|
137
|
+
description: 'One turn per item. Only the user turns are sent, in order, each as soon as the reply before it ends; an assistant turn is kept for whoever reads the case. `waitSeconds` on a user turn keeps a gap before it (`- { user: "Still there?", waitSeconds: 60 }` sends it a minute after the reply before). A case of more than one turn runs on this computer only: the workspace\'s evaluation datasets hold single questions, so `frontera local sync push` proposes the file\'s single-question cases and keeps the rest here.',
|
|
138
138
|
items: {
|
|
139
139
|
type: 'object',
|
|
140
140
|
properties: {
|
|
@@ -299,7 +299,7 @@ export function startStudioServer(opts: StudioServerOptions) {
|
|
|
299
299
|
if (!(file instanceof Blob)) return json({ error: 'send the document as the multipart field "file"' }, 400)
|
|
300
300
|
const name = checkUpload((file as File).name ?? '', file.size)
|
|
301
301
|
const replace = url.searchParams.get('replace') === '1'
|
|
302
|
-
// Replacing: the runtime's copy goes first, so the apply uploads the new content
|
|
302
|
+
// Replacing: the runtime's copy goes first, so the apply uploads the new content as a document the runtime lacks.
|
|
303
303
|
if (replace && id && documentInFolder(project.root, folder, name)) {
|
|
304
304
|
const call = serviceCall()
|
|
305
305
|
if (call) {
|
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
type LocalConfig,
|
|
33
33
|
type LocalSecrets,
|
|
34
34
|
} from './local-env'
|
|
35
|
+
import { runtimeKeyCustodianProblem } from './local-key-custodian'
|
|
35
36
|
import { localEmbeddingsReachable, providerEnv } from './local-keys'
|
|
36
37
|
import { applyProviderSetup, readProviderSetup, rememberSlugs, writeProviderSetup } from './local-providers'
|
|
37
38
|
import type { PortBlock } from './local-ports'
|
|
@@ -484,6 +485,7 @@ async function startService(
|
|
|
484
485
|
running.children.set('service', child)
|
|
485
486
|
const ok = await waitForHttp(`http://127.0.0.1:${ports.service}/health`, 120_000, () => child.exitCode === null)
|
|
486
487
|
if (!ok) throw new Error('the service did not become healthy; see `frontera local logs service`')
|
|
488
|
+
await refuseBrokenKeyCustodian(ports)
|
|
487
489
|
set('service', { status: 'running', pid: child.pid, startedAt: new Date().toISOString() })
|
|
488
490
|
keepDocs(project, { bundle: customerDocsBundle(config.platformSource) }, log)
|
|
489
491
|
refreshCheckoutDocs(project, config.platformSource, log)
|
|
@@ -508,10 +510,21 @@ async function startService(
|
|
|
508
510
|
followLogs(name, log)
|
|
509
511
|
const ok = await waitForHttp(`http://127.0.0.1:${ports.service}/health`, 180_000)
|
|
510
512
|
if (!ok) throw new Error('the service did not become healthy; see `frontera local logs service`')
|
|
513
|
+
await refuseBrokenKeyCustodian(ports)
|
|
511
514
|
set('service', { status: 'running', container: name, startedAt: new Date().toISOString() })
|
|
512
515
|
keepDocs(project, { fromContainer: (dest) => copyFromContainer(name, IMAGE_DOCS_PATH, dest) }, log)
|
|
513
516
|
}
|
|
514
517
|
|
|
518
|
+
/**
|
|
519
|
+
* A service whose encryption keys can't be read fails every secret and
|
|
520
|
+
* governed Action record later with a 500: the start stops here instead, with
|
|
521
|
+
* the fix (`local-key-custodian.ts`).
|
|
522
|
+
*/
|
|
523
|
+
async function refuseBrokenKeyCustodian(ports: PortBlock): Promise<void> {
|
|
524
|
+
const problem = await runtimeKeyCustodianProblem(`http://127.0.0.1:${ports.service}`)
|
|
525
|
+
if (problem) throw new Error(problem)
|
|
526
|
+
}
|
|
527
|
+
|
|
515
528
|
/** The release's documentation, for the solution's coding agents (`frontera mcp`). */
|
|
516
529
|
function keepDocs(project: LocalProject, from: { bundle?: string | null; fromContainer?: (dest: string) => boolean }, log: string): void {
|
|
517
530
|
if (!from.bundle && !from.fromContainer) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process'
|
|
2
|
-
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
|
|
2
|
+
import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs'
|
|
3
3
|
import { join } from 'node:path'
|
|
4
4
|
|
|
5
5
|
import { parseDocument } from 'yaml'
|
|
@@ -28,6 +28,11 @@ export interface SyncLink {
|
|
|
28
28
|
/** Names shown in Forge and `sync status`; refreshed on connect. */
|
|
29
29
|
workspaceName?: string
|
|
30
30
|
organizationName?: string
|
|
31
|
+
/**
|
|
32
|
+
* A model this computer names (a local provider's, by catalog slug) and the
|
|
33
|
+
* workspace's model it goes as when proposed (`local-sync-portable.ts`).
|
|
34
|
+
*/
|
|
35
|
+
models?: Record<string, string>
|
|
31
36
|
}
|
|
32
37
|
|
|
33
38
|
export function projectFile(root: string): string {
|
|
@@ -40,6 +45,9 @@ export function readLink(root: string): SyncLink | null {
|
|
|
40
45
|
const doc = parseDocument(readFileSync(file, 'utf8')).toJS() as { sync?: Partial<SyncLink> } | null
|
|
41
46
|
const s = doc?.sync
|
|
42
47
|
if (!s || typeof s.platform !== 'string' || typeof s.workspace !== 'string') return null
|
|
48
|
+
const models = s.models && typeof s.models === 'object' && !Array.isArray(s.models)
|
|
49
|
+
? Object.fromEntries(Object.entries(s.models as Record<string, unknown>).filter((e): e is [string, string] => typeof e[1] === 'string' && !!e[0] && !!e[1]))
|
|
50
|
+
: {}
|
|
43
51
|
return {
|
|
44
52
|
platform: s.platform,
|
|
45
53
|
workspace: s.workspace,
|
|
@@ -47,9 +55,41 @@ export function readLink(root: string): SyncLink | null {
|
|
|
47
55
|
blueprint: s.blueprint === 'organization' ? 'organization' : 'workspace',
|
|
48
56
|
workspaceName: typeof s.workspaceName === 'string' ? s.workspaceName : undefined,
|
|
49
57
|
organizationName: typeof s.organizationName === 'string' ? s.organizationName : undefined,
|
|
58
|
+
...(Object.keys(models).length ? { models } : {}),
|
|
50
59
|
}
|
|
51
60
|
}
|
|
52
61
|
|
|
62
|
+
const mapped = new Map<string, { mtimeMs: number; models: Record<string, string> }>()
|
|
63
|
+
|
|
64
|
+
/** The project's model mapping (`sync.models`), empty when it has none or no link; read once per change of the file. */
|
|
65
|
+
export function modelMap(root: string): Record<string, string> {
|
|
66
|
+
const file = projectFile(root)
|
|
67
|
+
let mtimeMs = -1
|
|
68
|
+
try {
|
|
69
|
+
mtimeMs = statSync(file).mtimeMs
|
|
70
|
+
} catch {
|
|
71
|
+
return {}
|
|
72
|
+
}
|
|
73
|
+
const known = mapped.get(file)
|
|
74
|
+
if (known && known.mtimeMs === mtimeMs) return known.models
|
|
75
|
+
const models = readLink(root)?.models ?? {}
|
|
76
|
+
mapped.set(file, { mtimeMs, models })
|
|
77
|
+
return models
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Map a local model to a workspace model when proposed, or (`workspace` null) stop mapping it. */
|
|
81
|
+
export function setModelMapping(root: string, local: string, workspace: string | null): SyncLink {
|
|
82
|
+
const link = readLink(root)
|
|
83
|
+
if (!link) throw new Error('the project is not linked to a workspace')
|
|
84
|
+
const models = { ...(link.models ?? {}) }
|
|
85
|
+
if (workspace) models[local] = workspace
|
|
86
|
+
else delete models[local]
|
|
87
|
+
const next: SyncLink = { ...link, models }
|
|
88
|
+
if (!Object.keys(models).length) delete next.models
|
|
89
|
+
writeLink(root, next)
|
|
90
|
+
return next
|
|
91
|
+
}
|
|
92
|
+
|
|
53
93
|
/** Write the link into `frontera.yaml`, keeping the rest of the file and its comments. */
|
|
54
94
|
export function writeLink(root: string, link: SyncLink | null): void {
|
|
55
95
|
const file = projectFile(root)
|
|
@@ -58,11 +98,12 @@ export function writeLink(root: string, link: SyncLink | null): void {
|
|
|
58
98
|
if (!link) {
|
|
59
99
|
doc.delete('sync')
|
|
60
100
|
} else {
|
|
61
|
-
const value: Record<string,
|
|
101
|
+
const value: Record<string, unknown> = { platform: link.platform, workspace: link.workspace }
|
|
62
102
|
if (link.workspaceName) value.workspaceName = link.workspaceName
|
|
63
103
|
if (link.organization) value.organization = link.organization
|
|
64
104
|
if (link.organizationName) value.organizationName = link.organizationName
|
|
65
105
|
value.blueprint = link.blueprint
|
|
106
|
+
if (link.models && Object.keys(link.models).length) value.models = link.models
|
|
66
107
|
const existing = doc.get('sync', true) as { commentBefore?: string } | undefined
|
|
67
108
|
const node = doc.createNode(value) as unknown as { commentBefore?: string }
|
|
68
109
|
node.commentBefore = existing?.commentBefore ?? ' The platform workspace this project pulls from (`frontera local sync`). Keys are in your keychain, not here.'
|