@frontera-sdk/cli 1.52.45 → 1.52.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/apps-api.ts +3 -1
- package/src/api/automation-api.ts +32 -0
- package/src/api/credential-failure.ts +1 -1
- package/src/api/governed-action-api.ts +24 -0
- package/src/api/media-api.ts +24 -0
- package/src/api/platform-api.ts +13 -0
- package/src/binding-registry.ts +9 -2
- package/src/blueprint/compile.ts +3 -1
- package/src/blueprint/ontology-edit-plan.ts +71 -4
- package/src/blueprint/org-name-check.ts +64 -0
- package/src/blueprint/tree.ts +28 -0
- package/src/commands/action/deploy.ts +22 -5
- package/src/commands/action/index-commands.ts +2 -0
- package/src/commands/action/retire.ts +60 -0
- package/src/commands/agent/compose.ts +1 -0
- package/src/commands/agent/index-commands.ts +42 -0
- package/src/commands/app/deploy.ts +1 -1
- package/src/commands/app/dev.ts +2 -2
- package/src/commands/app/save.ts +1 -1
- package/src/commands/auth/add.ts +5 -1
- package/src/commands/auth/current.ts +6 -0
- package/src/commands/auth/index-commands.ts +3 -0
- package/src/commands/auth/login.ts +84 -0
- package/src/commands/auth/use.ts +21 -2
- package/src/commands/automation/bindings.ts +54 -10
- package/src/commands/automation/index-commands.ts +2 -3
- package/src/commands/blueprint/bind.ts +12 -1
- package/src/commands/blueprint/declarative.ts +6 -2
- package/src/commands/conversation/index-commands.ts +66 -0
- package/src/commands/conversation/local-runtime.ts +18 -0
- package/src/commands/local/check.ts +57 -11
- package/src/commands/local/data.ts +75 -7
- package/src/commands/local/dev.ts +77 -9
- package/src/commands/local/embeddings.ts +74 -0
- package/src/commands/local/index-commands.ts +133 -11
- package/src/commands/local/new.ts +78 -0
- package/src/commands/local/publish.ts +34 -0
- package/src/commands/local/steps.ts +105 -0
- package/src/commands/local/sync-request.ts +69 -23
- package/src/commands/local/sync.ts +2 -0
- package/src/commands/media/index-commands.ts +58 -3
- package/src/commands/registry.ts +11 -3
- package/src/commands/trace/index-commands.ts +56 -0
- package/src/commands/workspace/index-commands.ts +14 -7
- package/src/config.ts +4 -1
- package/src/dev-broker.ts +15 -1
- package/src/device-login.ts +143 -0
- package/src/exit.ts +36 -8
- package/src/flag-help.ts +30 -0
- package/src/forge/claude-adapter.ts +17 -4
- package/src/forge/permission-rules.ts +158 -0
- package/src/forge/touches.ts +7 -6
- package/src/function-outside-imports.ts +13 -2
- package/src/function-source.ts +90 -0
- package/src/harness.ts +4 -4
- package/src/key-reach.ts +64 -0
- package/src/local/local-actions.ts +15 -3
- package/src/local/local-apply-error.ts +26 -0
- package/src/local/local-apps.ts +9 -3
- package/src/local/local-blueprint-schemas.ts +15 -1
- package/src/local/local-confidential.ts +81 -0
- package/src/local/local-data-load.ts +130 -0
- package/src/local/local-dev-references.ts +24 -4
- package/src/local/local-embedding-setup.ts +62 -0
- package/src/local/local-jobs.ts +108 -0
- package/src/local/local-new.ts +148 -0
- package/src/local/local-org-settings.ts +1 -1
- package/src/local/local-person-steps.ts +340 -0
- package/src/local/local-personas.ts +16 -5
- package/src/local/local-portability.ts +22 -1
- package/src/local/local-publish.ts +26 -0
- package/src/local/local-request-evidence.ts +35 -6
- package/src/local/local-runtime-history.ts +4 -3
- package/src/local/local-said-once.ts +51 -0
- package/src/local/local-scaffold.ts +40 -2
- package/src/local/local-schemas.ts +60 -11
- package/src/local/local-stand-ins.ts +44 -0
- package/src/local/local-supervisor.ts +22 -0
- package/src/local/local-sync-models.ts +5 -2
- package/src/local/local-sync-names.ts +3 -0
- package/src/local/local-sync-portable.ts +64 -15
- package/src/local/local-sync-precheck.ts +30 -1
- package/src/local/local-sync-push.ts +81 -8
- package/src/local/local-sync-remote.ts +10 -7
- package/src/local/local-test-models.ts +173 -0
- package/src/local/local-test-runner.ts +232 -25
- package/src/local/local-test-scope.ts +174 -0
- package/src/local/local-ungranted-reads.ts +55 -0
- package/src/local/local-writers.ts +171 -0
- package/src/main.ts +75 -17
- package/src/mcp/browser/ax-snapshot.ts +13 -1
- package/src/mcp/browser/page-scripts.ts +1 -0
- package/src/mcp/mcp-conversations.ts +171 -0
- package/src/mcp/mcp-server.ts +4 -1
- package/src/mcp/mcp-traces.ts +7 -2
- package/src/mcp/tools/agent-tools.ts +94 -2
- package/src/mcp/tools/project-tools.ts +149 -13
- package/src/mcp/tools/test-tools.ts +64 -13
- package/src/output.ts +22 -7
- package/src/pack.ts +16 -0
- package/src/project-context.ts +18 -5
- package/src/project.ts +4 -0
- package/src/remote-guard.ts +52 -0
- package/src/vendor/kit-assets.json +11 -11
|
@@ -36,8 +36,13 @@ import { newSessionToken } from '../../local/local-studio-server'
|
|
|
36
36
|
import { migrateMcpConfigs } from '../../local/local-scaffold'
|
|
37
37
|
import { DEFAULT_REGISTRY, runSupervisor } from '../../local/local-supervisor'
|
|
38
38
|
import { compareWithLastRun, readLastRun, recordRun } from '../../local/local-test-baseline'
|
|
39
|
-
import { agentFilesProblem, caseConcurrency, caseCount, runCases, selectCaseFiles, waitedWords, type CaseFile, type CaseResult } from '../../local/local-test-runner'
|
|
39
|
+
import { agentFilesProblem, caseConcurrency, caseCount, caseRepeat, DEFAULT_PASS_RATE, judgedWords, passRateThreshold, repeatWords, runCases, selectCaseFiles, waitedWords, type CaseFile, type CaseResult } from '../../local/local-test-runner'
|
|
40
|
+
import { compareWithRun, comparisonWords, listTestRuns, readTestRun, runOnModel, saveTestRun, stuckWords } from '../../local/local-test-models'
|
|
41
|
+
import { modelSubstitutions, skippedCases, skippedWords } from '../../local/local-test-scope'
|
|
40
42
|
import { devServerLine, listDevServers, stopDevServers } from '../../mcp/mcp-apps'
|
|
43
|
+
import { restartJobs } from '../../local/local-jobs'
|
|
44
|
+
import { conversationTranscript } from '../../mcp/mcp-conversations'
|
|
45
|
+
import { runtimeLink } from '../../mcp/mcp-runtime'
|
|
41
46
|
import { caseTraces, evidenceLine, failingCaseEvidence } from '../../local/local-case-evidence'
|
|
42
47
|
import { notRunGroups, notRunWords } from '../../local/local-case-not-run'
|
|
43
48
|
import { findWorkflowTestFiles, recordWorkflowTestRun, runWorkflowTests, workflowFileState, type WorkflowTestFile, type WorkflowTestResult } from '../../local/local-workflow-tests'
|
|
@@ -54,12 +59,16 @@ import { localDocsMcp } from './docs-mcp'
|
|
|
54
59
|
import { localInit } from './init'
|
|
55
60
|
import { localKeyList, localKeyRemove, localKeySet } from './key'
|
|
56
61
|
import { localModelAdd, localModelDefault, localModelRemove, localModelsAvailable, localProviderAdd, localProviderRemove, localProviders } from './providers'
|
|
62
|
+
import { localPublish } from './publish'
|
|
63
|
+
import { localFmt, localNew } from './new'
|
|
64
|
+
import { localEmbeddings } from './embeddings'
|
|
57
65
|
import { localRemove } from './remove'
|
|
58
66
|
import { localSchema } from './schema'
|
|
59
67
|
import { localSync } from './sync'
|
|
60
68
|
import { keylessEmbeddingsStatus, KEYLESS_RETRIEVAL_NOTE } from '../../local/local-org-settings'
|
|
61
69
|
import { withFix } from '../../local/fix-words'
|
|
62
70
|
import { isKeyCustodianProblem, KEY_CUSTODIAN_FIX, runtimeKeyCustodianProblem } from '../../local/local-key-custodian'
|
|
71
|
+
import { readLock, recentWriters, writerWords } from '../../local/local-writers'
|
|
63
72
|
|
|
64
73
|
/**
|
|
65
74
|
* `frontera local …`: Frontera's runtimes on this machine, for FDEs and the coding agents working with them.
|
|
@@ -353,6 +362,53 @@ export const localStop: Command = {
|
|
|
353
362
|
run: stop,
|
|
354
363
|
}
|
|
355
364
|
|
|
365
|
+
/**
|
|
366
|
+
* `frontera local restart jobs`: the jobs engine alone, back (X2). The
|
|
367
|
+
* service, Studio and the App dev servers keep running, and the runs the
|
|
368
|
+
* engine lost are marked failed so they stop holding the queue.
|
|
369
|
+
*/
|
|
370
|
+
async function restart(ctx: CommandContext) {
|
|
371
|
+
const project = localProject(ctx.cwd)
|
|
372
|
+
const what = ctx.positional[0]
|
|
373
|
+
if (what !== 'jobs') throw new UsageError(what ? `\`local restart\` restarts the jobs engine only, not ${what}` : 'say what to restart: jobs', '`frontera local restart jobs`; `frontera local start` brings back anything else that is down')
|
|
374
|
+
const state = readState(project)
|
|
375
|
+
if (!state || state.phase !== 'ready' || !state.supervisorPid || !isAlive(state.supervisorPid)) {
|
|
376
|
+
throw new CliError('the local runtime is not running', { code: 'LOCAL_NOT_RUNNING', hint: 'start it with `frontera local start`, which starts the jobs engine too' })
|
|
377
|
+
}
|
|
378
|
+
let restarted: Awaited<ReturnType<typeof restartJobs>>
|
|
379
|
+
try {
|
|
380
|
+
restarted = await ctx.output.task('Restarting the jobs engine', () => restartJobs(project, state, 'restarted with `frontera local restart jobs`'))
|
|
381
|
+
} catch (err) {
|
|
382
|
+
throw new CliError(`the jobs engine did not restart: ${(err as Error).message}`, { code: 'LOCAL_JOBS_RESTART', hint: '`frontera local logs jobs` says why; `frontera local start` restarts the whole runtime and keeps its data' })
|
|
383
|
+
}
|
|
384
|
+
const dev = listDevServers(project).length
|
|
385
|
+
return {
|
|
386
|
+
data: { restarted: 'jobs', ms: restarted.ms, lostRuns: restarted.lost, ...(restarted.note ? { note: restarted.note } : {}), devServersKept: dev },
|
|
387
|
+
text: [
|
|
388
|
+
`The jobs engine is back (${restarted.ms} ms).${restarted.lost === null ? '' : restarted.lost ? ` ${restarted.lost} run${restarted.lost === 1 ? '' : 's'} it had open ${restarted.lost === 1 ? 'was' : 'were'} lost with it and marked failed: run ${restarted.lost === 1 ? 'it' : 'them'} again.` : ' No run was lost.'}`,
|
|
389
|
+
...(restarted.note ? [` note: ${restarted.note}`] : []),
|
|
390
|
+
`The service, Studio${dev ? ` and ${dev} App dev server${dev === 1 ? '' : 's'}` : ''} kept running.`,
|
|
391
|
+
].join('\n'),
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
export const localRestart: Command = {
|
|
396
|
+
meta: {
|
|
397
|
+
noun: 'local',
|
|
398
|
+
verb: 'restart',
|
|
399
|
+
args: [{ name: 'component', required: true, description: 'jobs: the jobs engine that runs workflows and background work' }],
|
|
400
|
+
flags: {},
|
|
401
|
+
summary: 'Restart the local runtime\'s jobs engine alone, keeping the service and App dev servers running, and fail the runs it lost',
|
|
402
|
+
details: [
|
|
403
|
+
'The jobs engine keeps its runs in memory: a run still open when it restarts has no executor any more, so it is marked failed, saying the engine restarted, and stops holding the queue. Run it again.',
|
|
404
|
+
'The runtime\'s supervisor does the same on its own when the engine stops answering on its port twice in a row (it checks every 30 seconds). `frontera local status` shows each part as it is now.',
|
|
405
|
+
],
|
|
406
|
+
examples: ['frontera local restart jobs'],
|
|
407
|
+
offline: true,
|
|
408
|
+
},
|
|
409
|
+
run: restart,
|
|
410
|
+
}
|
|
411
|
+
|
|
356
412
|
/**
|
|
357
413
|
* Each project object set's id on the runtime, by its file's reference, read
|
|
358
414
|
* where `frontera blueprint object-sets` reads them; null when the runtime is
|
|
@@ -415,6 +471,9 @@ async function status(ctx: CommandContext) {
|
|
|
415
471
|
const orphans = (state.phase === 'ready' && isAlive(state.supervisorPid) && state.credentials ? await projectOrphans(project, `http://127.0.0.1:${state.ports.service}`, state.credentials).catch(() => null) : null) ?? []
|
|
416
472
|
// Encryption keys the runtime can't read (data keys a hosted OpenBao locked): every secret and governed Action record fails until a reset.
|
|
417
473
|
const keyProblem = state.phase === 'ready' && isAlive(state.supervisorPid) ? await runtimeKeyCustodianProblem(`http://127.0.0.1:${state.ports.service}`) : null
|
|
474
|
+
// Who last changed the runtime or the files, one line per session, and the run holding the runtime (X9).
|
|
475
|
+
const writers = recentWriters(project.stateDir)
|
|
476
|
+
const lock = readLock(project.stateDir)
|
|
418
477
|
return {
|
|
419
478
|
data: {
|
|
420
479
|
...rest,
|
|
@@ -436,6 +495,8 @@ async function status(ctx: CommandContext) {
|
|
|
436
495
|
// On the runtime, not in the files: each with what fixes it (`local dev` archives an Agent or workflow whose file was deleted).
|
|
437
496
|
...(orphans.length ? { notInProject: orphans } : {}),
|
|
438
497
|
...(keyProblem ? { keyCustodian: { problem: keyProblem, fix: KEY_CUSTODIAN_FIX } } : {}),
|
|
498
|
+
writers,
|
|
499
|
+
...(lock ? { lock: { command: lock.command, session: lock.session, harness: lock.harness, since: lock.since } } : {}),
|
|
439
500
|
},
|
|
440
501
|
text: [
|
|
441
502
|
renderStatus({ ...state, components }, project, undefined, live, setIds, personas, secrets, liveApps ?? null, down),
|
|
@@ -443,6 +504,7 @@ async function status(ctx: CommandContext) {
|
|
|
443
504
|
...(stuck.length ? ['', 'Knowledge', ...stuck.map((k) => ` warning: ${k.problem}`)] : []),
|
|
444
505
|
...(orphans.length ? ['', 'On the runtime, not in the project', ...orphans.map((o) => ` warning: ${orphanLine(o)}`)] : []),
|
|
445
506
|
...(devServers.length ? ['', 'App dev servers', ...devServers.map((s) => ` ${devServerLine(s)}`)] : []),
|
|
507
|
+
...(lock || writers.length ? ['', 'Writers', ...(lock ? [` held by \`frontera ${lock.command}\` since ${lock.since.slice(11, 19)}: other commands that change the runtime wait for it`] : []), ...writers.map((w) => ` ${writerWords(w)}`)] : []),
|
|
446
508
|
].join('\n'),
|
|
447
509
|
}
|
|
448
510
|
}
|
|
@@ -586,27 +648,54 @@ async function test(ctx: CommandContext) {
|
|
|
586
648
|
}
|
|
587
649
|
const serviceUrl = `http://127.0.0.1:${state.ports.service}`
|
|
588
650
|
let concurrency: number | undefined
|
|
651
|
+
let repeat: number | undefined
|
|
652
|
+
let passRate: number | undefined
|
|
589
653
|
try {
|
|
590
654
|
concurrency = caseConcurrency(flagString(ctx, 'concurrency'))
|
|
655
|
+
repeat = caseRepeat(flagString(ctx, 'repeat'))
|
|
656
|
+
passRate = passRateThreshold(flagString(ctx, 'pass-rate'))
|
|
591
657
|
} catch (err) {
|
|
592
|
-
throw new UsageError((err as Error).message, 'frontera local test --
|
|
658
|
+
throw new UsageError((err as Error).message, 'frontera local test --repeat 5 --pass-rate 0.8')
|
|
593
659
|
}
|
|
594
|
-
|
|
660
|
+
if (passRate !== undefined && !repeat) throw new UsageError('--pass-rate is the share of a repeated case’s runs that must pass: give --repeat too', 'frontera local test --repeat 5 --pass-rate 0.8')
|
|
661
|
+
// L1: another model for the run, and a kept run to compare with.
|
|
662
|
+
const model = flagString(ctx, 'model')?.trim() || null
|
|
663
|
+
if (model && flagBool(ctx, 'live')) throw new UsageError('--model runs the drafts on another model; --live tests the published versions as they are', 'frontera local test --model <slug>, without --live')
|
|
664
|
+
const baselineRef = flagString(ctx, 'baseline')?.trim()
|
|
665
|
+
const baseline = baselineRef ? readTestRun(project, baselineRef) : null
|
|
666
|
+
if (baselineRef && !baseline) {
|
|
667
|
+
const kept = listTestRuns(project)
|
|
668
|
+
throw new UsageError(`no kept test run ${baselineRef}`, kept.length ? `give one of: last, last-own, last-<model>, or a run id (${kept.slice(-5).join(', ')})` : 'every `frontera local test` keeps its run; run one first')
|
|
669
|
+
}
|
|
670
|
+
const runAll = async (): Promise<CaseResult[]> => total
|
|
595
671
|
? await runCases({
|
|
596
672
|
serviceUrl,
|
|
597
|
-
creds: state.credentials
|
|
673
|
+
creds: state.credentials!,
|
|
598
674
|
files,
|
|
599
675
|
live: flagBool(ctx, 'live'),
|
|
600
676
|
// The draft is what the files say, or the cases say what to run first.
|
|
601
677
|
staleDraft: (slug) => agentFilesProblem(project, state.credentials!.organizationId, slug),
|
|
602
678
|
concurrency,
|
|
679
|
+
...(repeat ? { repeat } : {}),
|
|
680
|
+
...(passRate ? { passRate } : {}),
|
|
603
681
|
onResult: (r) => {
|
|
604
682
|
// A case never sent after the run stopped is said once, below, not once each.
|
|
605
683
|
if (r.notRun?.skipped) return
|
|
606
|
-
ctx.output.note(` ${r.passed ? '✓' : r.notRun ? '–' : '✕'} ${r.agent} · ${r.name}${r.passed ? '' : r.notRun ? ` — couldn’t run: ${r.notRun.what}` : ` — ${r.reason}`}${r.passed && r.retried ? ' (run twice: the provider refused the first try for its limits)' : ''}${r.waited?.length ? ` (${waitedWords(r.waited)})` : ''} (${r.ms} ms)`)
|
|
684
|
+
ctx.output.note(` ${r.passed ? (r.repeat?.flaky ? '~' : '✓') : r.notRun ? '–' : '✕'} ${r.agent} · ${r.name}${r.passed ? '' : r.notRun ? ` — couldn’t run: ${r.notRun.what}` : ` — ${r.reason}`}${repeatWords(r)}${judgedWords(r)}${r.passed && r.retried ? ' (run twice: the provider refused the first try for its limits)' : ''}${r.waited?.length ? ` (${waitedWords(r.waited)})` : ''} (${r.ms} ms)`)
|
|
607
685
|
},
|
|
608
686
|
})
|
|
609
687
|
: []
|
|
688
|
+
let swap: { swapped: string[]; unchanged: string[] } | null = null
|
|
689
|
+
let results: CaseResult[]
|
|
690
|
+
if (model && total) {
|
|
691
|
+
const ran = await runOnModel(project, { serviceUrl, creds: state.credentials! }, files.map((f) => f.agent), model, async (on) => {
|
|
692
|
+
ctx.output.note(`Testing on ${model}: ${on.swapped.join(', ') || 'no Agent'}${on.unchanged.length ? ` (${on.unchanged.join(', ')} name no model and keep the runtime's default)` : ''}; each is put back after the run`)
|
|
693
|
+
return runAll()
|
|
694
|
+
})
|
|
695
|
+
swap = ran
|
|
696
|
+
results = ran.value
|
|
697
|
+
if (ran.stuck.length) ctx.output.note(` warning: ${stuckWords(ran.stuck, model)}`)
|
|
698
|
+
} else results = await runAll()
|
|
610
699
|
// What kept cases from running, once each, with its fix: never a verdict on the Agent.
|
|
611
700
|
const notRun = notRunGroups(results)
|
|
612
701
|
for (const g of notRun) {
|
|
@@ -629,15 +718,33 @@ async function test(ctx: CommandContext) {
|
|
|
629
718
|
// Every case's last-turn trace beside its conversation (`--json`), passing ones included; a failing one's came with its evidence.
|
|
630
719
|
const traces = await caseTraces(project, results, { known: evidence })
|
|
631
720
|
const comparison = compareWithLastRun(results, readLastRun(project))
|
|
632
|
-
|
|
721
|
+
// Another model's run is not the Agents' last run: a newly failing case there says nothing of the files.
|
|
722
|
+
if (results.length && !model) recordRun(project, results)
|
|
723
|
+
const kept = results.length ? saveTestRun(project, results, model) : null
|
|
724
|
+
const against = baseline ? compareWithRun(results, baseline) : null
|
|
633
725
|
// Each workflow case's last run, for Studio's Evaluations and Forge's home.
|
|
634
726
|
recordWorkflowTestRun(project, workflows)
|
|
635
727
|
const failed = [...results.filter((r) => !r.passed && !r.notRun).map((f) => `${f.agent} · ${f.name}`), ...workflows.filter((r) => !r.passed).map((f) => `workflow ${f.workflow} · ${f.name}`)]
|
|
636
728
|
const notRunCount = notRun.reduce((n, g) => n + g.cases.length, 0)
|
|
637
729
|
const passed = results.length + workflows.length - failed.length - notRunCount
|
|
638
730
|
const newly = comparison.newlyFailing.length ? `, ${comparison.newlyFailing.length} newly failing since the last run` : ''
|
|
639
|
-
|
|
640
|
-
const
|
|
731
|
+
// What the run left out, and why; the cases whose `judged` expectations weren't graded; Agents on another model here (X1).
|
|
732
|
+
const skipped = skippedCases(project.root, { ...(agent ? { agents: [agent] } : {}), ...(file ? { file } : {}), regression })
|
|
733
|
+
const flaky = results.filter((r) => r.repeat?.flaky)
|
|
734
|
+
const ungraded = results.filter((r) => r.judgedNotGraded)
|
|
735
|
+
const models = modelSubstitutions(project, state.credentials.organizationId, results.map((r) => r.agent))
|
|
736
|
+
for (const line of models) ctx.output.note(` note: ${line}`)
|
|
737
|
+
if (ungraded.length) ctx.output.note(` ${ungraded.length} case${ungraded.length === 1 ? '’s' : 's’'} judged expectations weren't graded: ${ungraded[0]!.judgedNotGraded}`)
|
|
738
|
+
if (against && baseline) {
|
|
739
|
+
ctx.output.note(` ${comparisonWords(against, baseline)}`)
|
|
740
|
+
for (const r of against.newlyFailing) ctx.output.note(` newly failing: ${r.agent} · ${r.name}`)
|
|
741
|
+
for (const r of against.fixed) ctx.output.note(` fixed: ${r.agent} · ${r.name}`)
|
|
742
|
+
}
|
|
743
|
+
const summary = `${passed} passed${flaky.length ? ` (${flaky.filter((r) => r.passed).length} flaky)` : ''}, ${failed.length} failed${model ? ` on ${model}` : ''}${model ? '' : newly}${notRun.length ? `, ${notRunWords(notRun)}` : ''}${repeat && repeat > 1 ? `; each case run ${repeat} times` : ''}${skippedWords(skipped.filter((s) => !/other Agents/.test(s.reason))) ? `; ${skippedWords(skipped.filter((s) => !/other Agents/.test(s.reason)))}` : ''}`
|
|
744
|
+
// `--transcripts`: each case's whole conversation, turn by turn, as `conversation get` reads it (N7).
|
|
745
|
+
const link = flagBool(ctx, 'transcripts') ? runtimeLink(project) : null
|
|
746
|
+
const transcripts = typeof link === 'object' && link ? await Promise.all(results.map((r) => (r.conversationId ? conversationTranscript(link, r.conversationId).catch(() => null) : Promise.resolve(null)))) : []
|
|
747
|
+
const data = { passed, failed: failed.length, ...(model ? { model, ...(swap ? { onModel: swap.swapped, ownModel: swap.unchanged } : {}) } : {}), ...(kept ? { run: kept.id } : {}), ...(against && baseline ? { baseline: { run: baseline.id, model: baseline.model, newlyFailing: against.newlyFailing.map((r) => `${r.agent} · ${r.name}`), fixed: against.fixed.map((r) => `${r.agent} · ${r.name}`), stillFailing: against.stillFailing.map((r) => `${r.agent} · ${r.name}`) } } : {}), ...(repeat && repeat > 1 ? { repeat, passRate: passRate ?? DEFAULT_PASS_RATE, flaky: flaky.map((r) => `${r.agent} · ${r.name}`) } : {}), ...(skipped.length ? { skipped } : {}), ...(models.length ? { modelSubstitutions: models } : {}), ...(ungraded.length ? { judgedNotGraded: ungraded.map((r) => `${r.agent} · ${r.name}`) } : {}), ...(notRun.length ? { notRun: notRunCount, couldNotRun: notRun.map((g) => ({ what: `couldn’t run: ${g.what}`, fix: g.fix, cases: g.cases, ...(g.skipped ? { notSent: g.skipped } : {}) })) } : {}), fixed: comparison.fixed.length, results: results.map((r, i) => ({ ...r, ...traces.get(i), ...evidence.get(i), ...(transcripts[i] ? { transcript: transcripts[i]!.turns } : {}) })), workflows }
|
|
641
748
|
if (failed.length || notRun.length) {
|
|
642
749
|
// Only cases that couldn't run: the run did not pass, and the cause is the provider's or the runtime's, not the Agent's.
|
|
643
750
|
const onlyNotRun = !failed.length
|
|
@@ -662,14 +769,24 @@ export const localTest: Command = {
|
|
|
662
769
|
noun: 'local',
|
|
663
770
|
verb: 'test',
|
|
664
771
|
args: [],
|
|
665
|
-
flags: { agent: 'string', workflow: 'string', file: 'string', live: 'boolean', regression: 'boolean', concurrency: 'string' },
|
|
666
|
-
flagHelp: {
|
|
772
|
+
flags: { agent: 'string', workflow: 'string', file: 'string', live: 'boolean', regression: 'boolean', concurrency: 'string', transcripts: 'boolean', repeat: 'string', 'pass-rate': 'string', model: 'string', baseline: 'string' },
|
|
773
|
+
flagHelp: {
|
|
774
|
+
model: 'run the Agents’ cases on this catalog model: each Agent’s draft is put on it for the run and back after',
|
|
775
|
+
baseline: 'compare with a kept run: last, last-own (the newest on the Agents’ own models), last-<model>, or a run id',
|
|
776
|
+
file: 'run only this case file, by its path from the project root: agents/<slug>/evals/<name>.yaml or workflows/tests/<slug>.yaml',
|
|
777
|
+
transcripts: 'with --json, each case’s whole conversation: every turn, tool call, procedure and reply check',
|
|
778
|
+
repeat: 'run each Agent case N times (1 to 10) and report its pass rate; one that passes some runs is flaky',
|
|
779
|
+
'pass-rate': 'with --repeat, the share of runs a flaky case must pass (default 0.8, or 80%)',
|
|
780
|
+
},
|
|
667
781
|
summary: 'Run the project\'s Agent test cases and workflow tests against the local runtime',
|
|
668
782
|
details: [
|
|
669
783
|
'`--file` runs one case file, by its path from the project root: an Agent\'s (agents/<slug>/evals/<name>.yaml) or a workflow\'s (workflows/tests/<slug>.yaml), the same file MCP `test_run` takes. A path that names no case file is refused.',
|
|
784
|
+
'`--repeat N` runs each Agent case N times and reports how many runs passed. A case that passes some runs and fails others is flaky: it passes when it passes at least `--pass-rate` of its runs (default 80%), and fails below that. Scenario cases run once.',
|
|
785
|
+
'`--model <slug>` compares models: the cases run with each Agent in scope on that catalog model, its draft swapped for the run and put back after (the files never change), and the run doesn\'t count as the Agents\' last run. Every run is kept under .frontera/local/test-runs/ with its model; `--baseline last-own` (or `last`, `last-<model>`, a run id) says which cases this run newly fails, fixed or still fails against it.',
|
|
786
|
+
'A case\'s `judged` expectations are graded on the local runtime by the expectation judge, on the model `frontera local model-default <model> --for tests` chose, once its other checks pass. When the runtime has no grading model it can use, the case says so and is graded on the rest.',
|
|
670
787
|
'A workflow test runs the workflow for real on the local runtime: a governed Action one of its steps submits (an action step, or a Function step\'s code) is applied and changes the runtime\'s records when the Action runs here; one whose effect would reach another system is switched off here. The writes stay; `frontera local check` rolls back its own tests\' writes, and `frontera local snapshot save` / `restore` do it by hand. The workspace is never touched.',
|
|
671
788
|
],
|
|
672
|
-
examples: ['frontera local test', 'frontera local test --agent claims-triage', 'frontera local test --workflow claim-intake', 'frontera local test --file agents/claims-triage/evals/refunds.yaml', 'frontera local test --live', 'frontera local test --regression', 'frontera local test --concurrency 1'],
|
|
789
|
+
examples: ['frontera local test', 'frontera local test --agent claims-triage', 'frontera local test --agent claims-triage --repeat 5', 'frontera local test --model claude-haiku-5-5 --baseline last-own', 'frontera local test --workflow claim-intake', 'frontera local test --file agents/claims-triage/evals/refunds.yaml', 'frontera local test --live', 'frontera local test --regression', 'frontera local test --concurrency 1'],
|
|
673
790
|
offline: true,
|
|
674
791
|
},
|
|
675
792
|
run: test,
|
|
@@ -701,6 +818,7 @@ export const localCommands: readonly Command[] = [
|
|
|
701
818
|
localInit,
|
|
702
819
|
localStart,
|
|
703
820
|
localStop,
|
|
821
|
+
localRestart,
|
|
704
822
|
localStatus,
|
|
705
823
|
localLogs,
|
|
706
824
|
localReset,
|
|
@@ -708,6 +826,10 @@ export const localCommands: readonly Command[] = [
|
|
|
708
826
|
localUi,
|
|
709
827
|
localDev,
|
|
710
828
|
localTest,
|
|
829
|
+
localPublish,
|
|
830
|
+
localNew,
|
|
831
|
+
localFmt,
|
|
832
|
+
localEmbeddings,
|
|
711
833
|
localCheck,
|
|
712
834
|
localKeySet,
|
|
713
835
|
localKeyList,
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { CliError, UsageError } from '../../errors'
|
|
2
|
+
import { formatProject, newAgentFiles, writeNewFiles } from '../../local/local-new'
|
|
3
|
+
import { localProject } from '../../local/local-project'
|
|
4
|
+
import { flagBool, flagString, type Command } from '../types'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* `frontera local new agent <slug>` (Forge improvement plan L8): a new
|
|
8
|
+
* Agent's folder from the scaffold's starter, or a copy of another Agent
|
|
9
|
+
* with `--like`. Only files are written; `frontera local dev` applies them.
|
|
10
|
+
*/
|
|
11
|
+
export const localNew: Command = {
|
|
12
|
+
meta: {
|
|
13
|
+
noun: 'local',
|
|
14
|
+
verb: 'new',
|
|
15
|
+
args: [
|
|
16
|
+
{ name: 'kind', required: true, description: '`agent`' },
|
|
17
|
+
{ name: 'slug', required: true, description: 'The new Agent’s slug: its folder under agents/' },
|
|
18
|
+
],
|
|
19
|
+
flags: { like: 'string', name: 'string', description: 'string', model: 'string', customer: 'boolean' },
|
|
20
|
+
flagHelp: {
|
|
21
|
+
like: 'start from another Agent’s files (instructions, bindings, behavior settings, cases), by its slug',
|
|
22
|
+
name: 'the Agent’s display name; defaults to the slug in words',
|
|
23
|
+
description: 'what the Agent does, as people see it',
|
|
24
|
+
model: 'the model it runs on, by its catalog slug (`frontera local models`)',
|
|
25
|
+
customer: 'it serves customers: write behavior settings (operating.json); an internal Agent has none',
|
|
26
|
+
},
|
|
27
|
+
summary: 'Write a new Agent’s files into the project, from the starter or a copy of another Agent',
|
|
28
|
+
details: ['Writes files only, and never over one that is there; `frontera local dev` applies them to the local runtime.'],
|
|
29
|
+
examples: ['frontera local new agent billing-help --name "Billing help"', 'frontera local new agent billing-help --like support-desk', 'frontera local new agent intake --customer --model gpt-5-mini'],
|
|
30
|
+
offline: true,
|
|
31
|
+
},
|
|
32
|
+
async run(ctx) {
|
|
33
|
+
const [kind, slug] = ctx.positional
|
|
34
|
+
if (kind !== 'agent') throw new UsageError(kind ? `\`local new\` makes an agent, not "${kind}"` : 'say what to make', 'frontera local new agent <slug>')
|
|
35
|
+
if (!slug) throw new UsageError('name the new Agent', 'frontera local new agent <slug>')
|
|
36
|
+
const project = localProject(ctx.cwd)
|
|
37
|
+
const like = flagString(ctx, 'like')
|
|
38
|
+
const files = newAgentFiles(project.root, slug, {
|
|
39
|
+
...(like ? { like } : {}),
|
|
40
|
+
...(flagString(ctx, 'name') ? { name: flagString(ctx, 'name') } : {}),
|
|
41
|
+
...(flagString(ctx, 'description') ? { description: flagString(ctx, 'description') } : {}),
|
|
42
|
+
...(flagString(ctx, 'model') ? { model: flagString(ctx, 'model') } : {}),
|
|
43
|
+
...(flagBool(ctx, 'customer') ? { customer: true } : {}),
|
|
44
|
+
})
|
|
45
|
+
const written = writeNewFiles(project.root, files)
|
|
46
|
+
return {
|
|
47
|
+
data: { agent: slug, like: like ?? null, written },
|
|
48
|
+
text: [`Wrote agents/${slug}/${like ? ` from agents/${like}/` : ''}:`, ...written.map((f) => ` ${f}`), `Next: \`frontera local dev\` applies it; \`frontera local test --agent ${slug}\` runs its cases.`].join('\n'),
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** `frontera local fmt [path…]` (L8): the project's JSON and YAML in one shape. */
|
|
54
|
+
export const localFmt: Command = {
|
|
55
|
+
meta: {
|
|
56
|
+
noun: 'local',
|
|
57
|
+
verb: 'fmt',
|
|
58
|
+
args: [{ name: 'path...', required: false, description: 'Only these files or folders, from the project root' }],
|
|
59
|
+
flags: { check: 'boolean' },
|
|
60
|
+
flagHelp: { check: 'write nothing; exit 1 when a file would change (for CI)' },
|
|
61
|
+
summary: 'Write the project’s JSON and YAML files in one shape: two-space indents, a final newline, comments kept',
|
|
62
|
+
details: ['Reads agents/, workflows/, blueprint/, knowledge/, skills/, chat-apps/, themes/, datasets/, mcp/, apps/<slug>.json and frontera.yaml; an App’s own code is left alone. A file that can’t be read is named, with where, and left as it is.'],
|
|
63
|
+
examples: ['frontera local fmt', 'frontera local fmt agents/support-desk', 'frontera local fmt --check'],
|
|
64
|
+
offline: true,
|
|
65
|
+
},
|
|
66
|
+
async run(ctx) {
|
|
67
|
+
const project = localProject(ctx.cwd)
|
|
68
|
+
const check = flagBool(ctx, 'check')
|
|
69
|
+
const outcome = formatProject(project.root, { check, only: ctx.positional })
|
|
70
|
+
const lines = [
|
|
71
|
+
...(outcome.changed.length ? [`${check ? 'Would change' : 'Formatted'} ${outcome.changed.length} file${outcome.changed.length === 1 ? '' : 's'}:`, ...outcome.changed.map((f) => ` ${f}`)] : ['Every file is already in shape.']),
|
|
72
|
+
...(outcome.failed.length ? ['Could not read:', ...outcome.failed.map((f) => ` ${f.path}: ${f.error}`)] : []),
|
|
73
|
+
]
|
|
74
|
+
if (outcome.failed.length) throw new CliError(`${outcome.failed.length} file${outcome.failed.length === 1 ? '' : 's'} could not be read`, { code: 'USAGE', hint: lines.join('\n'), details: outcome })
|
|
75
|
+
if (check && outcome.changed.length) throw new CliError(`${outcome.changed.length} file${outcome.changed.length === 1 ? ' is' : 's are'} not formatted`, { code: 'FAILURE', hint: `${outcome.changed.join(', ')}: run \`frontera local fmt\``, details: outcome })
|
|
76
|
+
return { data: outcome, text: lines.join('\n') }
|
|
77
|
+
},
|
|
78
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { CliError, UsageError } from '../../errors'
|
|
2
|
+
import { localProject } from '../../local/local-project'
|
|
3
|
+
import { publishLocalAgent } from '../../local/local-publish'
|
|
4
|
+
import { readState } from '../../local/local-state'
|
|
5
|
+
import { flagString, type Command } from '../types'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `frontera local publish <agent>`: publish the Agent's draft on this
|
|
9
|
+
* project's local runtime (Forge improvement plan L2), so a public website
|
|
10
|
+
* key, a Chat App or a channel there talks to it. It never touches a
|
|
11
|
+
* workspace.
|
|
12
|
+
*/
|
|
13
|
+
export const localPublish: Command = {
|
|
14
|
+
meta: {
|
|
15
|
+
noun: 'local',
|
|
16
|
+
verb: 'publish',
|
|
17
|
+
args: [{ name: 'agent', required: true, description: 'The Agent’s slug, its folder under agents/' }],
|
|
18
|
+
flags: { notes: 'string' },
|
|
19
|
+
flagHelp: { notes: 'what the version changes, kept with it' },
|
|
20
|
+
summary: 'Publish an Agent’s draft on the local runtime (public keys, Chat Apps and channels there use the published version)',
|
|
21
|
+
details: ['The local runtime only. The workspace’s Agent is published by a person, through a change request or on the platform.'],
|
|
22
|
+
examples: ['frontera local publish support-desk', 'frontera local publish support-desk --notes "New greeting"'],
|
|
23
|
+
offline: true,
|
|
24
|
+
},
|
|
25
|
+
async run(ctx) {
|
|
26
|
+
const slug = ctx.positional[0]
|
|
27
|
+
if (!slug) throw new UsageError('name the Agent', 'frontera local publish <agent>')
|
|
28
|
+
const project = localProject(ctx.cwd)
|
|
29
|
+
const state = readState(project)
|
|
30
|
+
if (!state || state.phase !== 'ready' || !state.credentials) throw new CliError('the local runtime is not running', { code: 'LOCAL_NOT_RUNNING', hint: 'run `frontera local start` first' })
|
|
31
|
+
const done = await publishLocalAgent({ serviceUrl: `http://127.0.0.1:${state.ports.service}`, creds: state.credentials }, slug, flagString(ctx, 'notes'))
|
|
32
|
+
return { data: done, text: `Published Agent ${slug} on the local runtime. Its public keys, Chat Apps and channels there use this version; the workspace is unchanged.` }
|
|
33
|
+
},
|
|
34
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { UsageError } from '../../errors'
|
|
2
|
+
import { FRONTERA_READ_ONLY_COMMANDS } from '../../forge/permission-rules'
|
|
3
|
+
import { addPersonStep, clearSettledSteps, CREDENTIAL_WORDS, CREDENTIALS, deliveryGroups, readPersonSteps, runStepChecks, settlePersonStep, type PersonStep } from '../../local/local-person-steps'
|
|
4
|
+
import { runSelf } from '../../mcp/mcp-cli'
|
|
5
|
+
import { localProject } from '../../local/local-project'
|
|
6
|
+
import { flagBool, flagString, type Command } from '../types'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* `frontera local steps`: the steps only the person takes (Forge improvement
|
|
10
|
+
* plan N4), kept in `.frontera/sync/person-steps.json`. The coding agent adds
|
|
11
|
+
* them (here, or with the MCP tool `person_step`); Forge shows them as one
|
|
12
|
+
* delivery card with **Run in my terminal**; the person runs each one and it
|
|
13
|
+
* is marked done. Nothing here runs a step.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const ACTIONS = ['list', 'add', 'done', 'dismiss', 'reopen', 'check', 'clear'] as const
|
|
17
|
+
|
|
18
|
+
function stepLines(steps: PersonStep[]): string[] {
|
|
19
|
+
return steps.flatMap((s, n) => [
|
|
20
|
+
` ${n + 1}. ${s.command ?? `${s.changes} (in the product)`}${s.check ? ' · checks itself' : ''}`,
|
|
21
|
+
` ${s.why} Changes: ${s.changes} Needs ${CREDENTIAL_WORDS[s.credential] ?? s.credential}. (${s.id})`,
|
|
22
|
+
])
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const localSteps: Command = {
|
|
26
|
+
meta: {
|
|
27
|
+
noun: 'local',
|
|
28
|
+
verb: 'steps',
|
|
29
|
+
args: [
|
|
30
|
+
{ name: 'action', required: false, description: ACTIONS.join(' | ') },
|
|
31
|
+
{ name: 'arg', required: false, description: 'add: the `frontera` command; done, dismiss, reopen: the step id' },
|
|
32
|
+
],
|
|
33
|
+
flags: { why: 'string', changes: 'string', credential: 'string', delivery: 'string', all: 'boolean' },
|
|
34
|
+
summary: 'The steps only the person takes (publishing, promoting, applying, granting, setting a secret), prepared for them to run',
|
|
35
|
+
details: [
|
|
36
|
+
'A step is one `frontera` command on one line, with why it is needed (`--why`), what it changes (`--changes`) and the credential it needs (`--credential`, one of: ' + CREDENTIALS.join(', ') + '). `--delivery` groups steps into one card, in the order they were added.',
|
|
37
|
+
'A step never chains commands and never carries a secret: a value is piped in by the person (`… --from -`). Forge shows the pending steps as a delivery card, and Run in my terminal types the command into a shell without pressing Enter.',
|
|
38
|
+
'done marks a step the person ran; dismiss one that is no longer needed; clear takes the settled ones off the list. `--all` lists the settled ones too.',
|
|
39
|
+
'An applied change request adds what it left for a person (deploying a new Action, binding a dataset, a Plugin’s secrets) as steps of its delivery, Pending in workspace in Forge. check runs each such step’s read (a read-only command, `--json`) and marks the ones it proves done; Forge runs it on a timer.',
|
|
40
|
+
],
|
|
41
|
+
examples: [
|
|
42
|
+
'frontera local steps',
|
|
43
|
+
'frontera local steps add "<command>" --why … --changes … --credential admin',
|
|
44
|
+
'frontera local steps done ps_1a2b3c4d5e',
|
|
45
|
+
],
|
|
46
|
+
offline: true,
|
|
47
|
+
},
|
|
48
|
+
async run(ctx) {
|
|
49
|
+
const action = (ctx.positional[0] ?? 'list') as (typeof ACTIONS)[number]
|
|
50
|
+
if (!ACTIONS.includes(action)) throw new UsageError(`\`${action}\` is not a steps action`, `use one of: ${ACTIONS.join(', ')}`)
|
|
51
|
+
const project = localProject(ctx.cwd)
|
|
52
|
+
if (action === 'add') {
|
|
53
|
+
const command = ctx.positional[1]
|
|
54
|
+
if (!command) throw new UsageError('missing the command', 'frontera local steps add "frontera app promote leads" --why "…" --changes "…" --credential "workspace admin"')
|
|
55
|
+
let added
|
|
56
|
+
try {
|
|
57
|
+
added = addPersonStep(project.root, {
|
|
58
|
+
command,
|
|
59
|
+
why: flagString(ctx, 'why') ?? '',
|
|
60
|
+
changes: flagString(ctx, 'changes') ?? '',
|
|
61
|
+
credential: flagString(ctx, 'credential') ?? '',
|
|
62
|
+
delivery: flagString(ctx, 'delivery') ?? null,
|
|
63
|
+
})
|
|
64
|
+
} catch (err) {
|
|
65
|
+
throw new UsageError(`not recorded: ${(err as Error).message}`, 'a step is one `frontera` command with --why, --changes and --credential')
|
|
66
|
+
}
|
|
67
|
+
return { data: added, text: `${added.updated ? 'Updated' : 'Added'} ${added.step.id}: ${added.step.command}\nForge shows it on the project's delivery card; the person runs it.` }
|
|
68
|
+
}
|
|
69
|
+
if (action === 'done' || action === 'dismiss' || action === 'reopen') {
|
|
70
|
+
const id = ctx.positional[1]
|
|
71
|
+
if (!id) throw new UsageError('missing the step id', '`frontera local steps` lists them')
|
|
72
|
+
let step
|
|
73
|
+
try {
|
|
74
|
+
step = settlePersonStep(project.root, id, action === 'done' ? 'done' : action === 'dismiss' ? 'dismissed' : 'pending')
|
|
75
|
+
} catch (err) {
|
|
76
|
+
throw new UsageError((err as Error).message, '`frontera local steps --all` lists every step')
|
|
77
|
+
}
|
|
78
|
+
return { data: { step }, text: `${step.id}: ${step.status}` }
|
|
79
|
+
}
|
|
80
|
+
if (action === 'check') {
|
|
81
|
+
const result = await runStepChecks(project.root, async (args) => {
|
|
82
|
+
const r = await runSelf(args, { cwd: project.root, timeoutMs: 60_000 })
|
|
83
|
+
return r.ok ? r.data : null
|
|
84
|
+
}, FRONTERA_READ_ONLY_COMMANDS)
|
|
85
|
+
const lines = [
|
|
86
|
+
result.checked ? `Checked ${result.checked} step${result.checked === 1 ? '' : 's'}; ${result.done.length} done.` : 'No pending step checks itself.',
|
|
87
|
+
...result.done.map((s) => ` ✓ ${s.changes}`),
|
|
88
|
+
...result.problems.map((p) => ` ! ${p.id}: ${p.problem}`),
|
|
89
|
+
]
|
|
90
|
+
return { data: result, text: lines.join('\n') }
|
|
91
|
+
}
|
|
92
|
+
if (action === 'clear') {
|
|
93
|
+
const removed = clearSettledSteps(project.root)
|
|
94
|
+
return { data: { removed }, text: removed ? `Took ${removed} settled step${removed === 1 ? '' : 's'} off the list.` : 'No settled steps to take off.' }
|
|
95
|
+
}
|
|
96
|
+
const steps = readPersonSteps(project.root)
|
|
97
|
+
const groups = deliveryGroups(steps)
|
|
98
|
+
const settled = steps.filter((s) => s.status !== 'pending')
|
|
99
|
+
const lines = groups.length
|
|
100
|
+
? groups.flatMap((g) => [g.delivery ? `${g.delivery}:` : 'Other steps:', ...stepLines(g.steps), ''])
|
|
101
|
+
: ['No steps wait on the person.', '']
|
|
102
|
+
if (flagBool(ctx, 'all') && settled.length) lines.push('Settled:', ...settled.map((s) => ` ${s.status === 'done' ? '✓' : '–'} ${s.command ?? s.changes} (${s.id})`))
|
|
103
|
+
return { data: { pending: groups, ...(flagBool(ctx, 'all') ? { settled } : {}) }, text: lines.join('\n').trimEnd() }
|
|
104
|
+
},
|
|
105
|
+
}
|