@frontera-sdk/cli 1.52.28 → 1.52.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -4
- package/src/commands/local/data.ts +60 -0
- package/src/commands/local/dev.ts +361 -18
- package/src/commands/local/index-commands.ts +11 -3
- package/src/commands/local/init.ts +88 -7
- package/src/commands/local/sync-request.ts +750 -0
- package/src/commands/local/sync.ts +941 -0
- package/src/commands/skill/bundle-commands.ts +9 -6
- package/src/errors.ts +4 -1
- package/src/exit.ts +3 -0
- package/src/flag-help.ts +48 -0
- package/src/local/local-actions.ts +306 -0
- package/src/local/local-adapt.ts +216 -0
- package/src/local/local-agent-files.ts +9 -2
- package/src/local/local-agent-grants.ts +125 -0
- package/src/local/local-agent-name.ts +71 -0
- package/src/local/local-applied.ts +334 -3
- package/src/local/local-apps.ts +156 -0
- package/src/local/local-blueprint-outline.ts +465 -0
- package/src/local/local-brand.ts +123 -0
- package/src/local/local-build-summary.ts +106 -0
- package/src/local/local-chat-apps.ts +170 -0
- package/src/local/local-dev-references.ts +1132 -0
- package/src/local/local-dev-sync.ts +171 -7
- package/src/local/local-docs.ts +53 -5
- package/src/local/local-env.ts +2 -0
- package/src/local/local-function-draft.ts +33 -0
- package/src/local/local-function-shared.ts +65 -0
- package/src/local/local-function-state.ts +34 -0
- package/src/local/local-knowledge-documents.ts +302 -0
- package/src/local/local-lessons.ts +278 -0
- package/src/local/local-library.ts +124 -0
- package/src/local/local-media-sets.ts +31 -0
- package/src/local/local-object-sets.ts +181 -0
- package/src/local/local-objects.ts +159 -0
- package/src/local/local-operating.ts +9 -5
- package/src/local/local-org-settings.ts +118 -0
- package/src/local/local-portability.ts +346 -0
- package/src/local/local-project.ts +6 -2
- package/src/local/local-quality-checks.ts +171 -0
- package/src/local/local-scaffold.ts +14 -11
- package/src/local/local-schemas.ts +33 -2
- package/src/local/local-secrets.ts +306 -0
- package/src/local/local-stand-ins.ts +442 -0
- package/src/local/local-studio-server.ts +23 -2
- package/src/local/local-sync-base.ts +237 -0
- package/src/local/local-sync-client.ts +93 -0
- package/src/local/local-sync-link.ts +122 -0
- package/src/local/local-sync-merge.ts +395 -0
- package/src/local/local-sync-names.ts +169 -0
- package/src/local/local-sync-plan.ts +717 -0
- package/src/local/local-sync-push.ts +986 -0
- package/src/local/local-sync-refs.ts +221 -0
- package/src/local/local-sync-remote.ts +1482 -0
- package/src/local/local-sync-snapshot.ts +526 -0
- package/src/local/local-synthetic-data.ts +374 -0
- package/src/local/local-test-runner.ts +172 -14
- package/src/local/local-themes.ts +201 -0
- package/src/local/local-workflow-state.ts +131 -0
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
2
|
+
import { dirname, join } from 'node:path'
|
|
3
|
+
|
|
4
|
+
import { CliError } from '../errors'
|
|
5
|
+
import { writeFileAtomic } from '../fs-atomic'
|
|
6
|
+
import { hash, itemHash, safePath, SYNC_KINDS, type RemoteItem, type SyncKind, type SyncLock } from './local-sync-plan'
|
|
7
|
+
import { readOrgSettings, type OrgSettings } from './local-org-settings'
|
|
8
|
+
import type { Portability } from './local-portability'
|
|
9
|
+
import { readWorkspace, type SyncSource } from './local-sync-remote'
|
|
10
|
+
import { REF_KINDS, type Directory, type RefKind, type Unresolved } from './local-sync-refs'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The workspace as last fetched, kept beside the project the way git keeps
|
|
14
|
+
* `refs/remotes/origin`: `frontera local sync fetch` writes it, and status,
|
|
15
|
+
* the pull plan and Forge's card read it without calling the platform.
|
|
16
|
+
*
|
|
17
|
+
* .frontera/sync/remote.json cursor, when, the platform's version and capabilities, every item
|
|
18
|
+
* .frontera/sync/remote/ the items' files in project form, as a pull would write them
|
|
19
|
+
* .frontera/sync/base/ the files each item had when last pulled: the merge base
|
|
20
|
+
*
|
|
21
|
+
* None of it is committed (`.frontera/.gitignore`): the snapshot is
|
|
22
|
+
* re-fetched and the base re-read at the revisions `frontera.lock` records.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
export const SYNC_DIR = join('.frontera', 'sync')
|
|
26
|
+
const REMOTE_JSON = join(SYNC_DIR, 'remote.json')
|
|
27
|
+
const REMOTE_DIR = join(SYNC_DIR, 'remote')
|
|
28
|
+
const BASE_DIR = join(SYNC_DIR, 'base')
|
|
29
|
+
/** Binary files that come with an item (a skill's assets), apart from its text files. */
|
|
30
|
+
const ASSETS_DIR = join(SYNC_DIR, 'remote-assets')
|
|
31
|
+
|
|
32
|
+
/** Kinds the index lists as one entry, read whole: a project holds them as many files cut from one read. */
|
|
33
|
+
const GROUP_KINDS: readonly SyncKind[] = ['blueprint', 'evaluation', 'brand', 'library']
|
|
34
|
+
|
|
35
|
+
/** A full index (no cursor) at least this often, so deletions and changes without a timestamp are seen. */
|
|
36
|
+
export const FULL_INDEX_EVERY_MS = 60 * 60 * 1000
|
|
37
|
+
|
|
38
|
+
export interface SnapshotItem {
|
|
39
|
+
kind: SyncKind
|
|
40
|
+
name: string
|
|
41
|
+
title: string
|
|
42
|
+
id?: string
|
|
43
|
+
revision?: string
|
|
44
|
+
hash: string
|
|
45
|
+
files: string[]
|
|
46
|
+
version?: string
|
|
47
|
+
unresolved: Unresolved[]
|
|
48
|
+
notes: string[]
|
|
49
|
+
requires: string[]
|
|
50
|
+
/** Whether it runs on the local runtime, and how (§8): class, adaptations in words, and what it needs. */
|
|
51
|
+
portability?: Portability
|
|
52
|
+
/** A workflow's or Function's state on the platform: live, switched on. */
|
|
53
|
+
state?: RemoteItem['state']
|
|
54
|
+
/** Its binary files (a skill's assets), by project path, kept under `.frontera/sync/remote-assets/`. */
|
|
55
|
+
assets?: string[]
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface RemoteSnapshot {
|
|
59
|
+
version: 1
|
|
60
|
+
workspace: string
|
|
61
|
+
/** The index's cursor; null after a full read on a platform without the index. */
|
|
62
|
+
cursor: string | null
|
|
63
|
+
fetchedAt: string
|
|
64
|
+
/** When the index was last read whole, so deletions are known as of then. */
|
|
65
|
+
fullIndexAt: string | null
|
|
66
|
+
/** The platform has no sync index: every fetch reads the whole workspace. */
|
|
67
|
+
full: boolean
|
|
68
|
+
platformVersion: string | null
|
|
69
|
+
capabilities: string[]
|
|
70
|
+
/** Kinds the index covers for this key: a ref of one of them missing from the snapshot is deleted. */
|
|
71
|
+
kinds: SyncKind[]
|
|
72
|
+
/** Revisions of the kinds read whole (the Blueprint, evaluations). */
|
|
73
|
+
groups: Partial<Record<SyncKind, string>>
|
|
74
|
+
/** Items the index lists that hold nothing a project can file (a workflow without a definition), by revision, so they are not read again until they change. */
|
|
75
|
+
skipped: Record<string, string>
|
|
76
|
+
items: Record<string, SnapshotItem>
|
|
77
|
+
/** From the last read of content. */
|
|
78
|
+
warnings: string[]
|
|
79
|
+
notes: string[]
|
|
80
|
+
/**
|
|
81
|
+
* The workspace's names → ids for each kind a reference may point at, as of
|
|
82
|
+
* the last read, so the plan can tell a file that only gained names (an id
|
|
83
|
+
* now written `action/resetLine`) from a change.
|
|
84
|
+
*/
|
|
85
|
+
names?: Partial<Record<RefKind, Record<string, string>>>
|
|
86
|
+
/**
|
|
87
|
+
* Read with an organization key as well: it brings governed Actions (and,
|
|
88
|
+
* for older platforms, the Blueprint). A snapshot read with other keys is
|
|
89
|
+
* read again whole, so adding the key brings what it can read.
|
|
90
|
+
*/
|
|
91
|
+
organizationKey?: boolean
|
|
92
|
+
/** What this CLI reads of an item (`SNAPSHOT_READER`); absent: 1. */
|
|
93
|
+
reader?: number
|
|
94
|
+
/**
|
|
95
|
+
* The organization's timezone, AI settings and feature flags, read on every
|
|
96
|
+
* fetch (`/v1/sync/settings`), so `local dev` can take the timezone and say
|
|
97
|
+
* where the runtime differs. Absent on a platform without the route.
|
|
98
|
+
*/
|
|
99
|
+
settings?: OrgSettings
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Raised when the CLI starts reading more of an item than before, so a
|
|
104
|
+
* snapshot an older one took is read again whole instead of waiting for each
|
|
105
|
+
* item's revision to move. 2: an Agent's Context and data (its Blueprint grants).
|
|
106
|
+
* 3: a Plugin's policy and an Agent's tool settings, an Agent's permission
|
|
107
|
+
* rules and guardrails, a knowledge base's search settings, and whether a
|
|
108
|
+
* workflow is live and on. 4: the index's revisions now move when a workflow
|
|
109
|
+
* is switched on or off and when a Plugin's per-tool policy changes; before,
|
|
110
|
+
* those changes left the revision as it was, so a snapshot can hold them
|
|
111
|
+
* stale. 5: a Function never deployed (its editor draft) and its on/off
|
|
112
|
+
* switch, a Plugin's secret settings as declared empty placeholders, and a
|
|
113
|
+
* skill's assets. 6: saved object sets, and a workflow's Blueprint step
|
|
114
|
+
* naming its set (`objectSet/<name>`) instead of the platform's id; a
|
|
115
|
+
* pulled Agent no longer carries the legacy `functions` rows. 7: evaluation
|
|
116
|
+
* cases keep their pattern and tool-call checks, judged expectations, gold
|
|
117
|
+
* flag and scenarios, every item is read, and the regression dataset is
|
|
118
|
+
* marked. 8: an Agent's Context and data is read with the workspace key where
|
|
119
|
+
* the platform gives it, and one that can't be read is said
|
|
120
|
+
* (`blueprintUnread`); a skill's assets are read with the workspace key.
|
|
121
|
+
* 9: an evaluation case keeps the name a change request recorded for it.
|
|
122
|
+
* 10: a Plugin's description and icon. 11: Custom Apps, the brand and the
|
|
123
|
+
* library. 12: an Agent's object tools carry their object definitions,
|
|
124
|
+
* named by slug (`object/<slug>`). 13: a knowledge base's quality checks
|
|
125
|
+
* (`quality-checks.yaml`).
|
|
126
|
+
*/
|
|
127
|
+
export const SNAPSHOT_READER = 13
|
|
128
|
+
|
|
129
|
+
/** The directory the snapshot recorded, as the reference rewriter reads it. */
|
|
130
|
+
export function directoryOf(snap: RemoteSnapshot | null): Directory | undefined {
|
|
131
|
+
if (!snap?.names) return undefined
|
|
132
|
+
return Object.fromEntries(Object.entries(snap.names).map(([k, v]) => [k, new Map(Object.entries(v ?? {}))])) as Directory
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function namesOf(directory: Directory): RemoteSnapshot['names'] {
|
|
136
|
+
return Object.fromEntries(REF_KINDS.flatMap((k) => (directory[k] ? [[k, Object.fromEntries([...directory[k]!].sort(([a], [b]) => a.localeCompare(b)))]] : [])))
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* What the snapshot holds, in one value: every item's ref, revision and
|
|
141
|
+
* content hash. A preview records it; applying a plan whose snapshot has
|
|
142
|
+
* moved since (a fetch brought changes) plans again first. A fetch that
|
|
143
|
+
* brought nothing keeps it, so it is not stale for nothing.
|
|
144
|
+
*/
|
|
145
|
+
export function snapshotId(snap: RemoteSnapshot): string {
|
|
146
|
+
return hash(Object.keys(snap.items).sort().map((r) => `${r}\0${snap.items[r]!.revision ?? ''}\0${snap.items[r]!.hash}`).join('\n'))
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function readSnapshot(root: string): RemoteSnapshot | null {
|
|
150
|
+
const file = join(root, REMOTE_JSON)
|
|
151
|
+
if (!existsSync(file)) return null
|
|
152
|
+
try {
|
|
153
|
+
const snap = JSON.parse(readFileSync(file, 'utf8')) as RemoteSnapshot
|
|
154
|
+
return snap && snap.version === 1 && snap.items && typeof snap.items === 'object' ? snap : null
|
|
155
|
+
} catch {
|
|
156
|
+
return null
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function underDir(root: string, dir: string, rel: string): string {
|
|
161
|
+
// `safePath` refuses a path that leaves the project; the same rule inside the snapshot folder.
|
|
162
|
+
safePath(root, rel)
|
|
163
|
+
return safePath(join(root, dir), rel)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function readUnder(root: string, dir: string, rel: string): string | null {
|
|
167
|
+
const full = underDir(root, dir, rel)
|
|
168
|
+
try {
|
|
169
|
+
return existsSync(full) ? readFileSync(full, 'utf8') : null
|
|
170
|
+
} catch {
|
|
171
|
+
return null
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** The snapshot's items as the plan reads them; an item whose files went missing is left out. */
|
|
176
|
+
export function snapshotItems(root: string, snap: RemoteSnapshot): RemoteItem[] {
|
|
177
|
+
return Object.entries(snap.items).flatMap(([ref, s]) => {
|
|
178
|
+
const files: Record<string, string> = {}
|
|
179
|
+
for (const path of s.files) {
|
|
180
|
+
const content = readUnder(root, REMOTE_DIR, path)
|
|
181
|
+
if (content === null) return []
|
|
182
|
+
files[path] = content
|
|
183
|
+
}
|
|
184
|
+
const assets = (s.assets ?? []).map((path) => ({ path, from: underDir(root, ASSETS_DIR, path) }))
|
|
185
|
+
return [{ ref, kind: s.kind, name: s.name, title: s.title, files, version: s.version, id: s.id, revision: s.revision, unresolved: s.unresolved, notes: s.notes, requires: s.requires, ...(s.portability ? { portability: s.portability } : {}), ...(s.state ? { state: s.state } : {}), ...(assets.length ? { assets } : {}) }]
|
|
186
|
+
})
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Whether the fetched workspace runs governed Actions: an item needs one by
|
|
191
|
+
* name or by the platform's id, or names one the read could not. With a
|
|
192
|
+
* workspace key alone the Actions cannot be read (§15.1), so Forge asks for
|
|
193
|
+
* an organization key when this is true.
|
|
194
|
+
*/
|
|
195
|
+
export function usesGovernedActions(snap: RemoteSnapshot | null): boolean {
|
|
196
|
+
if (!snap) return false
|
|
197
|
+
return Object.values(snap.items).some(
|
|
198
|
+
(it) => it.portability?.needs.some((n) => n.kind === 'action' || n.kind === 'action-unnamed') || it.unresolved.some((u) => u.kind === 'action'),
|
|
199
|
+
)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** Whether a ref's absence from the snapshot means the workspace deleted it. */
|
|
203
|
+
export function knownGone(snap: RemoteSnapshot | null): (ref: string) => boolean {
|
|
204
|
+
if (!snap || snap.full || !snap.fullIndexAt) return () => false
|
|
205
|
+
const kinds = new Set(snap.kinds)
|
|
206
|
+
return (ref) => kinds.has(ref.split('/')[0] as SyncKind) && !(ref in snap.items)
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** `.frontera/` keeps the runtime's state out of git; the snapshot and the base stay out too. */
|
|
210
|
+
function ignoreSyncDir(root: string): void {
|
|
211
|
+
const ignore = join(root, '.frontera', '.gitignore')
|
|
212
|
+
mkdirSync(dirname(ignore), { recursive: true })
|
|
213
|
+
const text = existsSync(ignore) ? readFileSync(ignore, 'utf8') : ''
|
|
214
|
+
if (text.split(/\r?\n/).some((l) => l.trim() === 'sync/' || l.trim() === 'sync')) return
|
|
215
|
+
if (text) appendFileSync(ignore, `${text.endsWith('\n') ? '' : '\n'}sync/\n`)
|
|
216
|
+
else writeFileSync(ignore, 'local/\nsync/\n')
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function toSnapshotItem(item: RemoteItem, revision?: string): SnapshotItem {
|
|
220
|
+
return {
|
|
221
|
+
kind: item.kind,
|
|
222
|
+
name: item.name,
|
|
223
|
+
title: item.title,
|
|
224
|
+
...(item.id ? { id: item.id } : {}),
|
|
225
|
+
...(revision ?? item.revision ? { revision: revision ?? item.revision } : {}),
|
|
226
|
+
hash: itemHash(item.files),
|
|
227
|
+
files: Object.keys(item.files).sort(),
|
|
228
|
+
...(item.version ? { version: item.version } : {}),
|
|
229
|
+
unresolved: item.unresolved,
|
|
230
|
+
notes: item.notes,
|
|
231
|
+
requires: item.requires ?? [],
|
|
232
|
+
...(item.portability ? { portability: item.portability } : {}),
|
|
233
|
+
...(item.state ? { state: item.state } : {}),
|
|
234
|
+
...(item.assets?.length ? { assets: item.assets.map((a) => a.path).sort() } : {}),
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Write the snapshot: the changed items' files, the removed items' files
|
|
240
|
+
* gone (unless another item holds the path), then `remote.json` last and
|
|
241
|
+
* atomically, so a reader sees the old snapshot or the new one.
|
|
242
|
+
*/
|
|
243
|
+
function writeSnapshot(root: string, previous: RemoteSnapshot | null, next: RemoteSnapshot, changed: RemoteItem[]): void {
|
|
244
|
+
ignoreSyncDir(root)
|
|
245
|
+
const held = new Set(Object.values(next.items).flatMap((i) => i.files))
|
|
246
|
+
const heldAssets = new Set(Object.values(next.items).flatMap((i) => i.assets ?? []))
|
|
247
|
+
for (const item of changed) {
|
|
248
|
+
for (const [path, content] of Object.entries(item.files)) {
|
|
249
|
+
const full = underDir(root, REMOTE_DIR, path)
|
|
250
|
+
mkdirSync(dirname(full), { recursive: true })
|
|
251
|
+
writeFileSync(full, content)
|
|
252
|
+
}
|
|
253
|
+
for (const asset of item.assets ?? []) {
|
|
254
|
+
if (!asset.bytes) continue
|
|
255
|
+
const full = underDir(root, ASSETS_DIR, asset.path)
|
|
256
|
+
mkdirSync(dirname(full), { recursive: true })
|
|
257
|
+
writeFileSync(full, asset.bytes)
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
for (const old of Object.values(previous?.items ?? {})) {
|
|
261
|
+
for (const path of old.files) if (!held.has(path)) rmSync(underDir(root, REMOTE_DIR, path), { force: true })
|
|
262
|
+
for (const path of old.assets ?? []) if (!heldAssets.has(path)) rmSync(underDir(root, ASSETS_DIR, path), { force: true })
|
|
263
|
+
}
|
|
264
|
+
writeFileAtomic(join(root, REMOTE_JSON), `${JSON.stringify(next, null, 2)}\n`, 0o644)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Keep what a pull wrote as the base the next merge starts from; a path the item no longer has leaves the base. */
|
|
268
|
+
export function writeBase(root: string, items: RemoteItem[], previous: SyncLock | null): void {
|
|
269
|
+
ignoreSyncDir(root)
|
|
270
|
+
for (const item of items) {
|
|
271
|
+
for (const path of Object.keys(previous?.items[item.ref]?.files ?? {})) {
|
|
272
|
+
if (!(path in item.files)) rmSync(underDir(root, BASE_DIR, path), { force: true })
|
|
273
|
+
}
|
|
274
|
+
for (const [path, content] of Object.entries(item.files)) {
|
|
275
|
+
const full = underDir(root, BASE_DIR, path)
|
|
276
|
+
mkdirSync(dirname(full), { recursive: true })
|
|
277
|
+
writeFileSync(full, content)
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** A base file as last pulled, or null. */
|
|
283
|
+
export function readBase(root: string, path: string): string | null {
|
|
284
|
+
return readUnder(root, BASE_DIR, path)
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const CLEARED_JSON = join(SYNC_DIR, 'cleared.json')
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Conflicts a later workspace change cleared (the newer version merged
|
|
291
|
+
* cleanly with the files here, and the merge was written), by ref, with the
|
|
292
|
+
* base they left: the plan says so while the item's base is still that one.
|
|
293
|
+
*/
|
|
294
|
+
export type ClearedConflicts = Record<string, { at: string; base: string }>
|
|
295
|
+
|
|
296
|
+
export function readCleared(root: string): ClearedConflicts {
|
|
297
|
+
const file = join(root, CLEARED_JSON)
|
|
298
|
+
try {
|
|
299
|
+
const parsed = existsSync(file) ? (JSON.parse(readFileSync(file, 'utf8')) as ClearedConflicts) : {}
|
|
300
|
+
return parsed && typeof parsed === 'object' ? parsed : {}
|
|
301
|
+
} catch {
|
|
302
|
+
return {}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
export function markCleared(root: string, cleared: ClearedConflicts): void {
|
|
307
|
+
if (!Object.keys(cleared).length) return
|
|
308
|
+
ignoreSyncDir(root)
|
|
309
|
+
writeFileAtomic(join(root, CLEARED_JSON), `${JSON.stringify({ ...readCleared(root), ...cleared }, null, 2)}\n`, 0o644)
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** Forget base files: an item no longer tracked has no base. */
|
|
313
|
+
export function dropBase(root: string, paths: string[]): void {
|
|
314
|
+
for (const path of paths) rmSync(underDir(root, BASE_DIR, path), { force: true })
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// ── Fetch ────────────────────────────────────────────────────────────────────
|
|
318
|
+
|
|
319
|
+
interface Capabilities { version?: number; platformVersion?: string; features?: string[] }
|
|
320
|
+
interface IndexItem { ref: string; id: string; kind: string; revision: string; group?: boolean; deleted?: boolean }
|
|
321
|
+
interface IndexAnswer { cursor: string; full: boolean; kinds: string[]; items: IndexItem[] }
|
|
322
|
+
|
|
323
|
+
export interface FetchResult {
|
|
324
|
+
snapshot: RemoteSnapshot
|
|
325
|
+
/** Refs whose content was read this time. */
|
|
326
|
+
changed: string[]
|
|
327
|
+
/** Refs that left the snapshot. */
|
|
328
|
+
removed: string[]
|
|
329
|
+
/** The index was read whole (or the platform has none and everything was read). */
|
|
330
|
+
fullIndex: boolean
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
export interface FetchOptions {
|
|
334
|
+
blueprint?: 'workspace' | 'organization'
|
|
335
|
+
/** Read the whole index, and with it every deletion, even when the cursor is recent. */
|
|
336
|
+
full?: boolean
|
|
337
|
+
now?: Date
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const isKind = (k: string): k is SyncKind => (SYNC_KINDS as readonly string[]).includes(k)
|
|
341
|
+
|
|
342
|
+
const WARNING_LABEL: Partial<Record<SyncKind, [string, string]>> = {
|
|
343
|
+
agent: ['Agent', 'Agents'],
|
|
344
|
+
workflow: ['Workflow', 'Workflows'],
|
|
345
|
+
function: ['Function', 'Functions'],
|
|
346
|
+
skill: ['Skill', 'Skills'],
|
|
347
|
+
knowledge: ['Knowledge', 'Knowledge'],
|
|
348
|
+
plugin: ['Plugin', 'Plugins'],
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** Whether the read warned about this ref, or about its whole kind's list. */
|
|
352
|
+
export function readFailed(warnings: string[], ref: string): boolean {
|
|
353
|
+
const [kind, ...rest] = ref.split('/')
|
|
354
|
+
const label = WARNING_LABEL[kind as SyncKind]
|
|
355
|
+
if (!label) return false
|
|
356
|
+
const name = rest.join('/')
|
|
357
|
+
return warnings.some((w) => w.startsWith(`${label[0]} ${name}:`) || w.startsWith(`${label[1]}:`))
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/** A read that warned because the platform stopped answering (no answer, or a gateway's 502, 503 or 504) is not a partial view: it is a failed fetch. One item that fails (a 500) is retried on its own next time. */
|
|
361
|
+
function unanswered(warnings: string[]): string | null {
|
|
362
|
+
return warnings.find((w) => / did not answer: | answered 50[234]: /.test(w)) ?? null
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* The read, unless the platform stopped answering while it ran: then the
|
|
367
|
+
* fetch fails, and the snapshot stays as the last good fetch left it, instead
|
|
368
|
+
* of being written with what happened to come back (which reads as "deleted"
|
|
369
|
+
* or "up to date").
|
|
370
|
+
*/
|
|
371
|
+
function stopIfUnanswered<T extends { warnings: string[] }>(read: T): T {
|
|
372
|
+
const why = unanswered(read.warnings)
|
|
373
|
+
if (why) throw new CliError(`the workspace could not be read: ${why}`, { code: 'UNAVAILABLE', hint: 'the last fetch is kept; check the platform is running and reachable, then fetch again' })
|
|
374
|
+
return read
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* Bring the snapshot up to date. On a platform with the sync index: read the
|
|
379
|
+
* capabilities, the index (from the saved cursor, or whole when there is
|
|
380
|
+
* none, when `full`, or once an hour), then only the changed items' content.
|
|
381
|
+
* On an older platform: read the whole workspace, and say so (`full: true`).
|
|
382
|
+
*/
|
|
383
|
+
export async function fetchWorkspace(root: string, source: SyncSource, opts: FetchOptions = {}): Promise<FetchResult> {
|
|
384
|
+
const now = opts.now ?? new Date()
|
|
385
|
+
const stored = readSnapshot(root)
|
|
386
|
+
const previous = stored?.workspace === source.workspaceId ? stored : null
|
|
387
|
+
// Keys that read more (or less) than the snapshot's did, or a CLI that reads more of an
|
|
388
|
+
// item than the one that took it: everything is read again.
|
|
389
|
+
const keysChanged = !!previous && (!!previous.organizationKey !== !!source.organization || (previous.reader ?? 1) < SNAPSHOT_READER)
|
|
390
|
+
// A platform without the route answers 404 (`find` gives null) and is read the older way. One that does
|
|
391
|
+
// not answer, or fails (5xx), is not an older platform: the fetch stops and the last snapshot stays.
|
|
392
|
+
const caps = await source.workspace.find<Capabilities>('/v1/sync/capabilities').catch((err: Error) => {
|
|
393
|
+
const code = (err as { code?: string }).code
|
|
394
|
+
if (code === 'UNAUTHORIZED' || code === 'UNAVAILABLE') throw err
|
|
395
|
+
return null
|
|
396
|
+
})
|
|
397
|
+
const capabilities = caps?.features ?? []
|
|
398
|
+
const platformVersion = caps?.platformVersion ?? null
|
|
399
|
+
|
|
400
|
+
// Read on every fetch: small, and it has no revision of its own.
|
|
401
|
+
const settings = capabilities.includes('settings') ? await readOrgSettings(source.workspace) : null
|
|
402
|
+
if (!capabilities.includes('index')) {
|
|
403
|
+
const read = stopIfUnanswered(await readWorkspace(source, { blueprint: opts.blueprint }))
|
|
404
|
+
const items = Object.fromEntries(read.items.map((i) => [i.ref, toSnapshotItem(i)]))
|
|
405
|
+
const next: RemoteSnapshot = {
|
|
406
|
+
version: 1,
|
|
407
|
+
workspace: source.workspaceId,
|
|
408
|
+
cursor: null,
|
|
409
|
+
fetchedAt: now.toISOString(),
|
|
410
|
+
fullIndexAt: null,
|
|
411
|
+
full: true,
|
|
412
|
+
platformVersion,
|
|
413
|
+
capabilities,
|
|
414
|
+
kinds: [],
|
|
415
|
+
groups: {},
|
|
416
|
+
skipped: {},
|
|
417
|
+
items,
|
|
418
|
+
warnings: read.warnings,
|
|
419
|
+
notes: read.notes,
|
|
420
|
+
names: namesOf(read.directory),
|
|
421
|
+
organizationKey: !!source.organization,
|
|
422
|
+
reader: SNAPSHOT_READER,
|
|
423
|
+
...(settings ? { settings } : {}),
|
|
424
|
+
}
|
|
425
|
+
const changed = read.items.filter((i) => previous?.items[i.ref]?.hash !== itemHash(i.files))
|
|
426
|
+
writeSnapshot(root, previous, next, read.items)
|
|
427
|
+
return { snapshot: next, changed: changed.map((i) => i.ref), removed: Object.keys(previous?.items ?? {}).filter((r) => !(r in items)), fullIndex: true }
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
const recentFull = previous?.fullIndexAt && now.getTime() - Date.parse(previous.fullIndexAt) < FULL_INDEX_EVERY_MS
|
|
431
|
+
const since = !opts.full && !keysChanged && previous && !previous.full && previous.cursor && recentFull ? previous.cursor : null
|
|
432
|
+
const index = await source.workspace.get<IndexAnswer>(`/v1/sync/index${since ? `?since=${encodeURIComponent(since)}` : ''}`)
|
|
433
|
+
const kinds = index.kinds.filter(isKind)
|
|
434
|
+
const items: Record<string, SnapshotItem> = previous && !previous.full && !keysChanged ? { ...previous.items } : {}
|
|
435
|
+
const groups: Partial<Record<SyncKind, string>> = previous && !previous.full && !keysChanged ? { ...previous.groups } : {}
|
|
436
|
+
const skipped: Record<string, string> = previous && !previous.full && !keysChanged ? { ...previous.skipped } : {}
|
|
437
|
+
const removed = new Set<string>()
|
|
438
|
+
const drop = (ref: string) => {
|
|
439
|
+
if (ref in items) removed.add(ref)
|
|
440
|
+
delete items[ref]
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// What to read: items whose revision moved (or that the snapshot lacks), and kinds read whole whose revision moved.
|
|
444
|
+
const only = new Set<string>()
|
|
445
|
+
const readKinds = new Set<SyncKind>()
|
|
446
|
+
const regroup = new Set<SyncKind>()
|
|
447
|
+
const listed = new Set<string>()
|
|
448
|
+
for (const entry of index.items) {
|
|
449
|
+
if (!isKind(entry.kind)) continue
|
|
450
|
+
if (entry.group || GROUP_KINDS.includes(entry.kind)) {
|
|
451
|
+
if (groups[entry.kind] !== entry.revision) {
|
|
452
|
+
readKinds.add(entry.kind)
|
|
453
|
+
regroup.add(entry.kind)
|
|
454
|
+
}
|
|
455
|
+
continue
|
|
456
|
+
}
|
|
457
|
+
listed.add(entry.ref)
|
|
458
|
+
// A rename: the same platform id under another name leaves its old name.
|
|
459
|
+
for (const [ref, it] of Object.entries(items)) if (it.id === entry.id && ref !== entry.ref && it.kind === entry.kind) drop(ref)
|
|
460
|
+
const have = items[entry.ref]?.revision ?? skipped[entry.ref]
|
|
461
|
+
if (have === entry.revision) continue
|
|
462
|
+
only.add(entry.ref)
|
|
463
|
+
readKinds.add(entry.kind)
|
|
464
|
+
}
|
|
465
|
+
// A whole index lists everything there is: what it does not list, of a kind it covers, is gone.
|
|
466
|
+
if (index.full) {
|
|
467
|
+
const groupsListed = new Set(index.items.filter((e) => e.group || GROUP_KINDS.includes(e.kind as SyncKind)).map((e) => e.kind))
|
|
468
|
+
for (const [ref, it] of Object.entries(items)) {
|
|
469
|
+
if (!kinds.includes(it.kind)) continue
|
|
470
|
+
if (GROUP_KINDS.includes(it.kind) ? !groupsListed.has(it.kind) : !listed.has(ref)) drop(ref)
|
|
471
|
+
}
|
|
472
|
+
for (const kind of GROUP_KINDS) if (kinds.includes(kind) && !groupsListed.has(kind)) delete groups[kind]
|
|
473
|
+
for (const ref of Object.keys(skipped)) if (!listed.has(ref)) delete skipped[ref]
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
let read: Awaited<ReturnType<typeof readWorkspace>> | null = null
|
|
477
|
+
if (readKinds.size) {
|
|
478
|
+
// The Blueprint and evaluations are read whole whatever `only` says.
|
|
479
|
+
read = stopIfUnanswered(await readWorkspace(source, { kinds: readKinds, only, blueprint: opts.blueprint }))
|
|
480
|
+
}
|
|
481
|
+
const revisionOf = new Map(index.items.map((e) => [e.ref, e.revision]))
|
|
482
|
+
const got = read?.items ?? []
|
|
483
|
+
// A kind read whole replaces every item of that kind.
|
|
484
|
+
for (const kind of regroup) {
|
|
485
|
+
for (const [ref, it] of Object.entries(items)) if (it.kind === kind && !got.some((g) => g.ref === ref)) drop(ref)
|
|
486
|
+
const entry = index.items.find((e) => e.kind === kind)
|
|
487
|
+
if (entry) groups[kind] = entry.revision
|
|
488
|
+
}
|
|
489
|
+
for (const it of got) {
|
|
490
|
+
if (!regroup.has(it.kind) && !only.has(it.ref)) continue
|
|
491
|
+
// A kind read whole carries its group's revision (the Blueprint's release), so a pull records it
|
|
492
|
+
// in the lock and a lost base can be read again at it.
|
|
493
|
+
items[it.ref] = toSnapshotItem(it, regroup.has(it.kind) ? groups[it.kind] : revisionOf.get(it.ref))
|
|
494
|
+
removed.delete(it.ref)
|
|
495
|
+
delete skipped[it.ref]
|
|
496
|
+
}
|
|
497
|
+
// Listed but nothing came back and nothing failed: a thing with no files (a workflow without a
|
|
498
|
+
// definition). Remembered by revision so it is not read again until it changes; a failed read is retried.
|
|
499
|
+
for (const ref of only) {
|
|
500
|
+
if (got.some((g) => g.ref === ref) || readFailed(read?.warnings ?? [], ref)) continue
|
|
501
|
+
skipped[ref] = revisionOf.get(ref)!
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
const next: RemoteSnapshot = {
|
|
505
|
+
version: 1,
|
|
506
|
+
workspace: source.workspaceId,
|
|
507
|
+
cursor: index.cursor,
|
|
508
|
+
fetchedAt: now.toISOString(),
|
|
509
|
+
fullIndexAt: index.full ? now.toISOString() : previous?.fullIndexAt ?? null,
|
|
510
|
+
full: false,
|
|
511
|
+
platformVersion,
|
|
512
|
+
capabilities,
|
|
513
|
+
kinds,
|
|
514
|
+
groups,
|
|
515
|
+
skipped,
|
|
516
|
+
items,
|
|
517
|
+
warnings: read?.warnings ?? [],
|
|
518
|
+
notes: read ? read.notes : previous?.notes ?? [],
|
|
519
|
+
...(read ? { names: namesOf(read.directory) } : previous?.names ? { names: previous.names } : {}),
|
|
520
|
+
organizationKey: !!source.organization,
|
|
521
|
+
reader: SNAPSHOT_READER,
|
|
522
|
+
...(settings ? { settings } : previous?.settings ? { settings: previous.settings } : {}),
|
|
523
|
+
}
|
|
524
|
+
writeSnapshot(root, previous, next, got.filter((g) => g.ref in items))
|
|
525
|
+
return { snapshot: next, changed: got.filter((g) => g.ref in items).map((g) => g.ref), removed: [...removed], fullIndex: index.full }
|
|
526
|
+
}
|