@frontera-sdk/cli 1.52.38 → 1.52.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +6 -4
- package/src/adopt.ts +6 -2
- package/src/api/apps-api.ts +86 -4
- package/src/api/blueprint-authoring-api.ts +35 -11
- package/src/api/dataset-api.ts +29 -0
- package/src/api/governed-action-api.ts +90 -2
- package/src/api/platform-api.ts +87 -2
- package/src/api/workflow-api.ts +33 -2
- package/src/app-contract.ts +156 -0
- package/src/app-lock.ts +125 -0
- package/src/args.ts +14 -3
- package/src/auth-verify.ts +10 -0
- package/src/automation-template.ts +12 -1
- package/src/blueprint/action-rules.ts +516 -0
- package/src/blueprint/action-semantic-fit.ts +205 -0
- package/src/blueprint/compile.ts +51 -2
- package/src/blueprint/diff.ts +34 -2
- package/src/blueprint/metric-query.ts +215 -0
- package/src/blueprint/metric-rules.ts +122 -0
- package/src/blueprint/object-sets.ts +116 -0
- package/src/blueprint/ontology-edit-plan.ts +48 -4
- package/src/blueprint/projection.ts +24 -1
- package/src/blueprint/scaffold.ts +84 -42
- package/src/commands/action/available.ts +1 -1
- package/src/commands/action/criteria-refusal.ts +84 -0
- package/src/commands/action/decide.ts +5 -2
- package/src/commands/action/deploy.ts +13 -4
- package/src/commands/action/list.ts +134 -19
- package/src/commands/action/requests.ts +34 -6
- package/src/commands/action/submit.ts +28 -13
- package/src/commands/agent/index-commands.ts +69 -7
- package/src/commands/agent/resolve.ts +2 -0
- package/src/commands/app/checks.ts +44 -0
- package/src/commands/app/deploy.ts +243 -60
- package/src/commands/app/deployments.ts +97 -0
- package/src/commands/app/dev.ts +103 -11
- package/src/commands/app/init.ts +36 -3
- package/src/commands/app/promote.ts +45 -19
- package/src/commands/app/pull.ts +3 -1
- package/src/commands/app/rollback.ts +46 -0
- package/src/commands/app/sdk.ts +54 -8
- package/src/commands/app/unpin.ts +27 -0
- package/src/commands/app/versions.ts +1 -1
- package/src/commands/automation/bindings.ts +4 -2
- package/src/commands/automation/build-entry.ts +4 -1
- package/src/commands/automation/index-commands.ts +3 -1
- package/src/commands/automation/pull.ts +6 -1
- package/src/commands/automation/run.ts +34 -9
- package/src/commands/blueprint/bind.ts +2 -1
- package/src/commands/blueprint/declarative.ts +331 -31
- package/src/commands/blueprint/get.ts +6 -2
- package/src/commands/blueprint/grants.ts +64 -10
- package/src/commands/blueprint/metric.ts +108 -0
- package/src/commands/blueprint/object-sets.ts +71 -0
- package/src/commands/blueprint/query.ts +155 -31
- package/src/commands/dataset/index-commands.ts +156 -7
- package/src/commands/knowledge/index-commands.ts +122 -20
- package/src/commands/local/check.ts +657 -44
- package/src/commands/local/data.ts +33 -12
- package/src/commands/local/dev.ts +875 -88
- package/src/commands/local/docs-mcp.ts +10 -18
- package/src/commands/local/index-commands.ts +300 -63
- package/src/commands/local/init.ts +18 -5
- package/src/commands/local/key.ts +0 -3
- package/src/commands/local/providers.ts +0 -7
- package/src/commands/local/remove.ts +116 -0
- package/src/commands/local/schema.ts +3 -4
- package/src/commands/local/sdk-refresh.ts +59 -0
- package/src/commands/local/snapshot.ts +100 -0
- package/src/commands/local/sync-request.ts +320 -13
- package/src/commands/local/sync.ts +102 -5
- package/src/commands/mcp/serve.ts +34 -0
- package/src/commands/registry.ts +52 -8
- package/src/commands/types.ts +29 -0
- package/src/commands/workflow/archive.ts +46 -0
- package/src/commands/workflow/get.ts +2 -1
- package/src/commands/workflow/index-commands.ts +5 -0
- package/src/commands/workflow/push.ts +49 -2
- package/src/commands/workflow/run.ts +65 -7
- package/src/commands/workflow/runs.ts +9 -6
- package/src/commands/workflow/webhook-test.ts +166 -0
- package/src/config.ts +43 -6
- package/src/dev-broker.ts +52 -0
- package/src/exit.ts +103 -4
- package/src/flag-help.ts +77 -17
- package/src/forge/touches.ts +15 -12
- package/src/function-outside-imports.ts +89 -0
- package/src/harness.ts +42 -24
- package/src/heal.ts +112 -1
- package/src/local/fix-words.ts +12 -0
- package/src/local/function-bundle-recovery.ts +775 -0
- package/src/local/local-actions.ts +103 -10
- package/src/local/local-agent-action-grants.ts +210 -0
- package/src/local/local-agent-checks.ts +232 -0
- package/src/local/local-agent-grants.ts +29 -7
- package/src/local/local-agent-lean-plugins.ts +68 -0
- package/src/local/local-agent-missing-tools.ts +68 -0
- package/src/local/local-agent-name.ts +27 -0
- package/src/local/local-app-code.ts +191 -0
- package/src/local/local-app-dev-files.ts +186 -0
- package/src/local/local-app-dev-session.ts +88 -0
- package/src/local/local-app-lock-install.ts +114 -0
- package/src/local/local-app-pulled.ts +167 -0
- package/src/local/local-app-restore.ts +111 -0
- package/src/local/local-app-runtime.ts +68 -0
- package/src/local/local-app-sdk-paths.ts +149 -0
- package/src/local/local-app-sdk.ts +509 -0
- package/src/local/local-applied.ts +211 -21
- package/src/local/local-apply-history.ts +84 -0
- package/src/local/local-apply-summary.ts +86 -0
- package/src/local/local-apply-watchdog.ts +95 -0
- package/src/local/local-apps.ts +180 -7
- package/src/local/local-blueprint-errors.ts +198 -0
- package/src/local/local-blueprint-outline.ts +31 -12
- package/src/local/local-blueprint-schemas.ts +582 -0
- package/src/local/local-build-inputs.ts +48 -0
- package/src/local/local-build-summary.ts +495 -28
- package/src/local/local-case-evidence.ts +120 -0
- package/src/local/local-case-not-run.ts +153 -0
- package/src/local/local-chat-apps.ts +93 -16
- package/src/local/local-check-inherited.ts +83 -0
- package/src/local/local-cities.ts +89 -0
- package/src/local/local-component-health.ts +119 -0
- package/src/local/local-control.ts +34 -7
- package/src/local/local-db-snapshot.ts +348 -0
- package/src/local/local-dev-references.ts +346 -35
- package/src/local/local-dev-sync.ts +136 -7
- package/src/local/local-docker.ts +35 -1
- package/src/local/local-docs-search.ts +1 -1
- package/src/local/local-docs.ts +281 -2
- package/src/local/local-empty-workflows.ts +92 -0
- package/src/local/local-env.ts +16 -1
- package/src/local/local-file-checks.ts +104 -0
- package/src/local/local-function-ids.ts +141 -0
- package/src/local/local-function-recovery.ts +141 -0
- package/src/local/local-function-state.ts +87 -0
- package/src/local/local-keys.ts +7 -5
- package/src/local/local-knowledge-ingestion.ts +225 -0
- package/src/local/local-knowledge-uploads.ts +217 -0
- package/src/local/local-media-placeholders.ts +182 -0
- package/src/local/local-media-sets.ts +90 -10
- package/src/local/local-model-parameters.ts +89 -0
- package/src/local/local-object-sets.ts +18 -6
- package/src/local/local-operating.ts +35 -1
- package/src/local/local-org-settings.ts +205 -7
- package/src/local/local-orphans.ts +321 -0
- package/src/local/local-personas.ts +296 -0
- package/src/local/local-portability.ts +141 -16
- package/src/local/local-ports.ts +99 -7
- package/src/local/local-project-notices.ts +234 -0
- package/src/local/local-project-secrets.ts +133 -0
- package/src/local/local-project.ts +12 -0
- package/src/local/local-provider-refusal.ts +66 -0
- package/src/local/local-request-evidence.ts +112 -0
- package/src/local/local-runtime-health.ts +123 -0
- package/src/local/local-runtime-history.ts +50 -0
- package/src/local/local-runtime-target.ts +42 -0
- package/src/local/local-sample-data.ts +141 -0
- package/src/local/local-sample-remake.ts +138 -0
- package/src/local/local-scaffold.ts +336 -53
- package/src/local/local-schema-check.ts +148 -0
- package/src/local/local-schemas.ts +742 -25
- package/src/local/local-secret-carry.ts +107 -0
- package/src/local/local-secrets.ts +38 -8
- package/src/local/local-seed.ts +102 -26
- package/src/local/local-stand-ins.ts +1 -1
- package/src/local/local-state.ts +43 -2
- package/src/local/local-studio-server.ts +170 -7
- package/src/local/local-supervisor.ts +44 -6
- package/src/local/local-sync-names.ts +2 -0
- package/src/local/local-sync-plan.ts +11 -0
- package/src/local/local-sync-push.ts +10 -7
- package/src/local/local-sync-refs.ts +49 -3
- package/src/local/local-sync-remote.ts +343 -32
- package/src/local/local-sync-snapshot.ts +15 -10
- package/src/local/local-synthetic-data.ts +2116 -99
- package/src/local/local-test-baseline.ts +113 -0
- package/src/local/local-test-cases.ts +76 -1
- package/src/local/local-test-runner.ts +582 -91
- package/src/local/local-test-runs.ts +13 -5
- package/src/local/local-vendored-sdk.ts +183 -0
- package/src/local/local-workflow-state.ts +16 -3
- package/src/local/local-workflow-tests.ts +482 -0
- package/src/local/local-workspace-grants.ts +81 -0
- package/src/main.ts +50 -9
- package/src/mcp/a11y-scan.ts +55 -0
- package/src/mcp/browser/ax-snapshot.ts +181 -0
- package/src/mcp/browser/credentials.ts +78 -0
- package/src/mcp/browser/frames.ts +336 -0
- package/src/mcp/browser/headless-session.ts +413 -0
- package/src/mcp/browser/headless-tab.ts +575 -0
- package/src/mcp/browser/keys.ts +70 -0
- package/src/mcp/browser/page-scripts.ts +129 -0
- package/src/mcp/browser/wait.ts +73 -0
- package/src/mcp/forge-link.ts +126 -0
- package/src/mcp/headless.ts +199 -0
- package/src/mcp/mcp-apps.ts +435 -0
- package/src/mcp/mcp-chat.ts +226 -0
- package/src/mcp/mcp-cli.ts +53 -0
- package/src/mcp/mcp-protocol.ts +122 -0
- package/src/mcp/mcp-runtime.ts +113 -0
- package/src/mcp/mcp-server.ts +108 -0
- package/src/mcp/mcp-traces.ts +409 -0
- package/src/mcp/page-observation.ts +193 -0
- package/src/mcp/tools/agent-tools.ts +700 -0
- package/src/mcp/tools/app-tools.ts +241 -0
- package/src/mcp/tools/browser-tools.ts +310 -0
- package/src/mcp/tools/docs-tools.ts +32 -0
- package/src/mcp/tools/project-tools.ts +205 -0
- package/src/mcp/tools/test-tools.ts +339 -0
- package/src/mcp/tools/workflow-tools.ts +611 -0
- package/src/output.ts +1 -1
- package/src/pack.ts +8 -3
- package/src/packaging.ts +1 -1
- package/src/project-bootstrap.ts +107 -4
- package/src/project-context.ts +18 -1
- package/src/project.ts +27 -2
- package/src/provenance.ts +25 -0
- package/src/render-evidence.ts +65 -7
- package/src/template.ts +14 -8
- package/src/templates/next-app-files.ts +35 -6
- package/src/templates/next-skills.ts +61 -27
- package/src/vendor/kit-assets.json +6 -6
- package/src/vendor/sdk-sources.json +24 -21
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { connect } from 'node:net'
|
|
2
|
+
|
|
3
|
+
import { containerState } from './local-docker'
|
|
4
|
+
import { isAlive, type ComponentName, type ComponentState, type LocalState } from './local-state'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Whether each part of a running local runtime is up now, not what the state
|
|
8
|
+
* file said when it started: a container can exit under the supervisor (the
|
|
9
|
+
* container engine restarted, the laptop slept), and the state file still says
|
|
10
|
+
* `running`. `local status` shows what this finds, and `local start` brings
|
|
11
|
+
* back what is down instead of saying "Already running".
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export interface ComponentProbe {
|
|
15
|
+
container: (name: string) => ReturnType<typeof containerState>
|
|
16
|
+
port: (port: number) => Promise<boolean>
|
|
17
|
+
alive: (pid: number) => boolean
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** A part of the runtime that is down, and why, in words. `cache` is the service's Redis, not a listed component. */
|
|
21
|
+
export interface DownComponent {
|
|
22
|
+
name: ComponentName | 'cache'
|
|
23
|
+
why: string
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Whether something accepts a TCP connection on 127.0.0.1 at `port`, within `timeoutMs`. */
|
|
27
|
+
export function portAnswers(port: number, timeoutMs = 1500): Promise<boolean> {
|
|
28
|
+
return new Promise((resolvePromise) => {
|
|
29
|
+
const socket = connect({ host: '127.0.0.1', port })
|
|
30
|
+
const done = (ok: boolean) => {
|
|
31
|
+
socket.removeAllListeners()
|
|
32
|
+
socket.destroy()
|
|
33
|
+
resolvePromise(ok)
|
|
34
|
+
}
|
|
35
|
+
socket.setTimeout(timeoutMs, () => done(false))
|
|
36
|
+
socket.once('connect', () => done(true))
|
|
37
|
+
socket.once('error', () => done(false))
|
|
38
|
+
})
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const defaultProbe: ComponentProbe = {
|
|
42
|
+
container: (name) => {
|
|
43
|
+
try {
|
|
44
|
+
return containerState(name)
|
|
45
|
+
} catch {
|
|
46
|
+
// No docker on PATH: nothing of the runtime's runs.
|
|
47
|
+
return 'missing'
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
port: (port) => portAnswers(port),
|
|
51
|
+
alive: isAlive,
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const containerOf = (state: LocalState, c: ComponentState) => c.container ?? `frontera-${state.projectId}-${c.name}`
|
|
55
|
+
|
|
56
|
+
async function whyDown(state: LocalState, c: ComponentState, probe: ComponentProbe): Promise<string | null> {
|
|
57
|
+
if (c.kind === 'container') {
|
|
58
|
+
const now = probe.container(containerOf(state, c))
|
|
59
|
+
if (now === 'missing') return 'its container is gone'
|
|
60
|
+
if (now !== 'running') return 'its container exited'
|
|
61
|
+
} else if (c.kind === 'process') {
|
|
62
|
+
if (!c.pid || !probe.alive(c.pid)) return 'its process exited'
|
|
63
|
+
} else if (!state.supervisorPid || !probe.alive(state.supervisorPid)) {
|
|
64
|
+
return 'the supervisor exited'
|
|
65
|
+
}
|
|
66
|
+
if (c.port && !(await probe.port(c.port))) return `nothing answers on its port ${c.port}`
|
|
67
|
+
return null
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The runtime's components as they are now, and the ones that are down. Only
|
|
72
|
+
* a component the state file says is running is checked (one that is off,
|
|
73
|
+
* stopped or already failed says so itself), and only while the supervisor is
|
|
74
|
+
* up: a stopped runtime is stopped, nothing more to find. The service's cache
|
|
75
|
+
* is checked with Postgres, which it starts beside.
|
|
76
|
+
*/
|
|
77
|
+
export async function liveComponents(state: LocalState, probe: ComponentProbe = defaultProbe): Promise<{ components: ComponentState[]; down: DownComponent[] }> {
|
|
78
|
+
if (state.phase !== 'ready' || !state.supervisorPid || !probe.alive(state.supervisorPid)) return { components: state.components, down: [] }
|
|
79
|
+
const down: DownComponent[] = []
|
|
80
|
+
const components = await Promise.all(
|
|
81
|
+
state.components.map(async (c) => {
|
|
82
|
+
if (c.status !== 'running') return c
|
|
83
|
+
const why = await whyDown(state, c, probe)
|
|
84
|
+
if (!why) return c
|
|
85
|
+
down.push({ name: c.name, why })
|
|
86
|
+
return { ...c, status: 'failed' as const, detail: `${why}; ${BRING_BACK}` }
|
|
87
|
+
}),
|
|
88
|
+
)
|
|
89
|
+
const postgres = state.components.find((c) => c.name === 'postgres')
|
|
90
|
+
if (postgres?.status === 'running' && postgres.kind === 'container') {
|
|
91
|
+
const cache = probe.container(`frontera-${state.projectId}-cache`)
|
|
92
|
+
if (cache !== 'running') down.push({ name: 'cache', why: cache === 'missing' ? 'its container is gone' : 'its container exited' })
|
|
93
|
+
}
|
|
94
|
+
down.sort((a, b) => ORDER.indexOf(a.name) - ORDER.indexOf(b.name))
|
|
95
|
+
return { components, down }
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* What `local start` brings back on a runtime whose supervisor is up and
|
|
100
|
+
* ready: each component that is down now, and each one the supervisor saw
|
|
101
|
+
* fail (a native process that crashed). Null when nothing is down, or when it
|
|
102
|
+
* is not a ready runtime (one still starting is left to finish).
|
|
103
|
+
*/
|
|
104
|
+
export async function downToRestart(state: LocalState, probe: ComponentProbe = defaultProbe): Promise<DownComponent[] | null> {
|
|
105
|
+
if (state.phase !== 'ready' || !state.supervisorPid || !probe.alive(state.supervisorPid)) return null
|
|
106
|
+
const { down } = await liveComponents(state, probe)
|
|
107
|
+
for (const c of state.components) if (c.status === 'failed' && !down.some((d) => d.name === c.name)) down.push({ name: c.name, why: c.detail ?? 'it failed' })
|
|
108
|
+
return down.length ? down : null
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const ORDER: Array<DownComponent['name']> = ['postgres', 'cache', 'storage', 'service', 'jobs', 'runner', 'studio']
|
|
112
|
+
|
|
113
|
+
/** What brings a down component back, said on its line. */
|
|
114
|
+
export const BRING_BACK = '`frontera local start` brings it back (the data is kept)'
|
|
115
|
+
|
|
116
|
+
/** The header's phase when something is down: `down: postgres (its container exited), cache (…)`. */
|
|
117
|
+
export function downPhase(down: DownComponent[]): string {
|
|
118
|
+
return `not answering: ${down.map((d) => `${d.name} down (${d.why})`).join(', ')}`
|
|
119
|
+
}
|
|
@@ -4,12 +4,11 @@ import { join, resolve } from 'node:path'
|
|
|
4
4
|
|
|
5
5
|
import { credentialStore } from '../credential-store'
|
|
6
6
|
import { findContext, writeContext } from '../project-context'
|
|
7
|
-
import { fingerprint, putProfile } from '../profiles'
|
|
8
|
-
import { containersForProject,
|
|
7
|
+
import { fingerprint, getProfile, putProfile } from '../profiles'
|
|
8
|
+
import { containersForProject, removeRuntimeResources, stopContainer } from './local-docker'
|
|
9
9
|
import { stopProcess } from './local-process'
|
|
10
10
|
import type { LocalProject } from './local-project'
|
|
11
11
|
import { isAlive, logPath, readState, type LocalState } from './local-state'
|
|
12
|
-
import { volumeName } from './local-supervisor'
|
|
13
12
|
|
|
14
13
|
/**
|
|
15
14
|
* What the `frontera local` commands share: finding the platform source,
|
|
@@ -89,9 +88,7 @@ export async function stopRuntime(project: LocalProject): Promise<void> {
|
|
|
89
88
|
}
|
|
90
89
|
|
|
91
90
|
export function destroyRuntimeData(project: LocalProject): void {
|
|
92
|
-
|
|
93
|
-
removeVolume(volumeName(project))
|
|
94
|
-
removeVolume(`frontera-${project.id}-storage`)
|
|
91
|
+
removeRuntimeResources(project.id)
|
|
95
92
|
}
|
|
96
93
|
|
|
97
94
|
export const localProfileName = (project: LocalProject) => `local-${project.id}`.slice(0, 64)
|
|
@@ -137,7 +134,37 @@ export async function registerLocalProfile(project: LocalProject, state: LocalSt
|
|
|
137
134
|
})
|
|
138
135
|
await store.set(orgProfile, creds.orgKey)
|
|
139
136
|
const existing = findContext(project.root)
|
|
140
|
-
if (existing && existing.profile !== profile) return { profile, bound: false }
|
|
137
|
+
if (existing && existing.profile !== profile && !copiedLocalBinding(project, existing)) return { profile, bound: false }
|
|
141
138
|
writeContext(project.root, profile)
|
|
142
139
|
return { profile, bound: true }
|
|
143
140
|
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Whether a binding is one a copy of the project carried here: it is in this
|
|
144
|
+
* directory, this machine never made it, and it names another directory's
|
|
145
|
+
* local runtime (`local-<other id>`). Rebinding such a one to this
|
|
146
|
+
* directory's own runtime is safe (a local key reaches only a runtime on this
|
|
147
|
+
* machine) and is what the copy meant. A binding to anything else (a
|
|
148
|
+
* platform profile) is never replaced: that is the person's to choose.
|
|
149
|
+
*/
|
|
150
|
+
export function copiedLocalBinding(project: LocalProject, found: { profile: string; root: string; trusted: boolean }): boolean {
|
|
151
|
+
return !found.trusted && resolve(found.root) === resolve(project.root) && found.profile.startsWith('local-') && found.profile !== localProfileName(project)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Bind a copied project to its own local runtime (`copiedLocalBinding`), when
|
|
156
|
+
* that runtime's profile is registered here: the line that says so, or null
|
|
157
|
+
* when nothing changed.
|
|
158
|
+
*/
|
|
159
|
+
export function rebindCopiedProject(project: LocalProject): string | null {
|
|
160
|
+
let found: ReturnType<typeof findContext>
|
|
161
|
+
try {
|
|
162
|
+
found = findContext(project.root)
|
|
163
|
+
} catch {
|
|
164
|
+
return null
|
|
165
|
+
}
|
|
166
|
+
const own = localProfileName(project)
|
|
167
|
+
if (!found || !copiedLocalBinding(project, found) || !getProfile(own)) return null
|
|
168
|
+
writeContext(project.root, own)
|
|
169
|
+
return ` bound this directory to its own local runtime (${own}): .frontera/context.json named ${found.profile}, another directory's runtime, which it was copied from`
|
|
170
|
+
}
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
3
|
+
import { join } from 'node:path'
|
|
4
|
+
|
|
5
|
+
import { writeFileAtomic } from '../fs-atomic'
|
|
6
|
+
import { loadHashes } from './local-applied'
|
|
7
|
+
import { appDeployments, appsLostByRestore, appsLostWords, markAppsLostByRestore, type AppLostByRestore } from './local-app-restore'
|
|
8
|
+
import { execInContainer, queryInContainer } from './local-docker'
|
|
9
|
+
import { POSTGRES_DB, POSTGRES_USER } from './local-env'
|
|
10
|
+
import type { LocalProject } from './local-project'
|
|
11
|
+
import { carrySecrets, readSecrets, type QueryFn, type SecretsCarried } from './local-secret-carry'
|
|
12
|
+
import { isAlive } from './local-state'
|
|
13
|
+
import { containerName } from './local-supervisor'
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The runtime's database as the project's files applied it: a Postgres
|
|
17
|
+
* template copy (`frontera_baseline`) taken after an apply, so what the tests
|
|
18
|
+
* write (a workflow test always writes for real, as does an Agent case with
|
|
19
|
+
* `realActions` or a `decision`) is rolled back afterwards instead of growing
|
|
20
|
+
* the data on every `local check`.
|
|
21
|
+
*
|
|
22
|
+
* The baseline is keyed by the applied build: a digest of what `local dev`
|
|
23
|
+
* recorded as applied (`applied-hashes.json`). It is taken again only when
|
|
24
|
+
* that changed. A copy of the hash and build records is kept with it, so a
|
|
25
|
+
* restore puts the CLI's record of what is applied back with the database:
|
|
26
|
+
* anything applied after the baseline counts as not applied, and the next
|
|
27
|
+
* `local dev` applies it again.
|
|
28
|
+
*
|
|
29
|
+
* Only the platform database is copied. The managed datasets' catalog
|
|
30
|
+
* database, object storage and local traces are left as they are.
|
|
31
|
+
*
|
|
32
|
+
* Workspace secrets are configuration, not test data: a restore keeps them as
|
|
33
|
+
* they are now (`local-secret-carry.ts`: the rows and the data keys they are
|
|
34
|
+
* encrypted under, copied as stored), unless it is asked for the baseline's
|
|
35
|
+
* (`secrets: 'baseline'`, `frontera local snapshot restore --exact`).
|
|
36
|
+
*
|
|
37
|
+
* The service, job engine and runner keep running: connections to the
|
|
38
|
+
* database are refused for the second or two it takes (`ALLOW_CONNECTIONS
|
|
39
|
+
* false`), the open ones ended (`pg_terminate_backend`), and the pools
|
|
40
|
+
* reconnect. The service's cache (Redis) is emptied after a restore. A save
|
|
41
|
+
* or restore then waits (bounded) until the service's database-backed health
|
|
42
|
+
* route answers on every pooled connection it asks (`serviceAnswers`, given
|
|
43
|
+
* `waitForDatabaseReady`), so the first request after it does not land on a
|
|
44
|
+
* connection that was ended.
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
export const BASELINE_DB = `${POSTGRES_DB}_baseline`
|
|
48
|
+
/** The database psql connects to while it copies, drops and creates the platform's. */
|
|
49
|
+
const MAINTENANCE_DB = 'postgres'
|
|
50
|
+
|
|
51
|
+
export interface BaselineRecord {
|
|
52
|
+
version: 1
|
|
53
|
+
organizationId: string
|
|
54
|
+
database: string
|
|
55
|
+
baseline: string
|
|
56
|
+
/** The applied build the baseline is of (`buildHash`). */
|
|
57
|
+
buildHash: string
|
|
58
|
+
savedAt: string
|
|
59
|
+
saveMs: number
|
|
60
|
+
/** `applied-hashes.json` and `build.json` as they were when the baseline was taken; null when absent. */
|
|
61
|
+
applied: string | null
|
|
62
|
+
build: string | null
|
|
63
|
+
/**
|
|
64
|
+
* Each Custom App folder's deployment when the baseline was taken (its
|
|
65
|
+
* `.frontera/state.json`): a deployment in the database copy. A restore puts
|
|
66
|
+
* these back in the folders and marks what it took away (`local-app-restore.ts`).
|
|
67
|
+
* Absent on a baseline taken before it was kept.
|
|
68
|
+
*/
|
|
69
|
+
apps?: Record<string, string | null>
|
|
70
|
+
restoredAt?: string
|
|
71
|
+
restoreMs?: number
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface SnapshotDeps {
|
|
75
|
+
/** `docker exec <container> <cmd…>`: true when it exited 0. */
|
|
76
|
+
exec?: (container: string, cmd: string[]) => boolean
|
|
77
|
+
/** Whether the service reaches its database again after a save or restore (`waitForDatabaseReady`). */
|
|
78
|
+
serviceAnswers?: () => Promise<boolean>
|
|
79
|
+
now?: () => number
|
|
80
|
+
/** How long the pools get to see their ended connections closed (500 ms). */
|
|
81
|
+
settleMs?: number
|
|
82
|
+
/**
|
|
83
|
+
* `docker exec -i <container> <cmd…>` with stdin, and what it printed: reads
|
|
84
|
+
* and writes the secrets a restore keeps. Without it, a stand-in `exec`
|
|
85
|
+
* carries no secrets (the real one carries them through `docker`).
|
|
86
|
+
*/
|
|
87
|
+
query?: QueryFn
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** What a restore does with the workspace secrets: keeps the ones set now (the default), or puts back the baseline's. */
|
|
91
|
+
export type RestoreSecrets = 'keep' | 'baseline'
|
|
92
|
+
|
|
93
|
+
export interface SnapshotOutcome {
|
|
94
|
+
ok: boolean
|
|
95
|
+
/** What happened, in a line. */
|
|
96
|
+
note: string
|
|
97
|
+
ms: number
|
|
98
|
+
/** save: false when the stored baseline is already of this build. */
|
|
99
|
+
took?: boolean
|
|
100
|
+
/** restore: items applied after the baseline, now counted as not applied (Custom Apps deployed after it included). */
|
|
101
|
+
reapply?: number
|
|
102
|
+
/** restore: the Custom Apps deployed after the baseline, which the restored database does not have. */
|
|
103
|
+
appsLost?: AppLostByRestore[]
|
|
104
|
+
/** restore: the workspace secrets kept as they were before it, or `baseline` when the baseline's were put back. */
|
|
105
|
+
secrets?: (SecretsCarried & { mode: 'keep' }) | { mode: 'baseline' }
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** The secrets part of a restore's line: what was kept, what has to be set again, or that the baseline's are back. */
|
|
109
|
+
export function secretsWords(secrets: SnapshotOutcome['secrets']): string {
|
|
110
|
+
if (!secrets) return ''
|
|
111
|
+
if (secrets.mode === 'baseline') return 'the workspace secrets are as the baseline holds them (--exact)'
|
|
112
|
+
if (!secrets.ok) return `the workspace secrets couldn’t be kept (${secrets.problem ?? 'unknown'}), so they are as the baseline holds them: set any changed since again with \`frontera secret set\``
|
|
113
|
+
const kept = secrets.kept ? `kept the ${secrets.kept === 1 ? 'workspace secret as it was' : `${secrets.kept} workspace secrets as they were`}` : 'the workspace secrets are as they were (none set)'
|
|
114
|
+
return secrets.notKept.length ? `${kept}; set again: ${secrets.notKept.join(', ')} (its value would not decrypt in the restored database)` : kept
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const recordPath = (project: LocalProject) => join(project.stateDir, 'db-baseline.json')
|
|
118
|
+
const lockPath = (project: LocalProject) => join(project.stateDir, 'db-snapshot.lock')
|
|
119
|
+
const hashesFile = (project: LocalProject) => join(project.stateDir, 'applied-hashes.json')
|
|
120
|
+
const buildFile = (project: LocalProject) => join(project.stateDir, 'build.json')
|
|
121
|
+
|
|
122
|
+
function readText(path: string): string | null {
|
|
123
|
+
try {
|
|
124
|
+
return readFileSync(path, 'utf8')
|
|
125
|
+
} catch {
|
|
126
|
+
return null
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* What is applied to this runtime, as one digest: the hash record, in a stable
|
|
132
|
+
* order, and each Custom App's deployment. An App deployed since the baseline
|
|
133
|
+
* changes it, so `local check` takes the baseline again rather than rolling the
|
|
134
|
+
* App back out of the runtime after its tests.
|
|
135
|
+
*/
|
|
136
|
+
export function buildHash(project: LocalProject, organizationId: string): string {
|
|
137
|
+
const entries = [...loadHashes(project, organizationId)].sort(([a], [b]) => a.localeCompare(b))
|
|
138
|
+
const apps = Object.entries(appDeployments(project.root)).filter(([, d]) => d).sort(([a], [b]) => a.localeCompare(b))
|
|
139
|
+
const hash = createHash('sha256').update(organizationId).update('\0').update(JSON.stringify(entries))
|
|
140
|
+
// Only when there is one, so a project without Apps keeps the digest its baseline was taken under.
|
|
141
|
+
if (apps.length) hash.update('\0apps\0').update(JSON.stringify(apps))
|
|
142
|
+
return hash.digest('hex')
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** The stored baseline's record, for this organization (after a reset the runtime is another one). */
|
|
146
|
+
export function readBaseline(project: LocalProject, organizationId: string): BaselineRecord | null {
|
|
147
|
+
try {
|
|
148
|
+
const saved = JSON.parse(readFileSync(recordPath(project), 'utf8')) as Partial<BaselineRecord>
|
|
149
|
+
return saved.version === 1 && saved.organizationId === organizationId && typeof saved.buildHash === 'string' ? (saved as BaselineRecord) : null
|
|
150
|
+
} catch {
|
|
151
|
+
return null
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function writeBaseline(project: LocalProject, record: BaselineRecord): void {
|
|
156
|
+
writeFileAtomic(recordPath(project), `${JSON.stringify(record, null, 2)}\n`, 0o644)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Whether a save or restore holds the database now: an apply that fails in
|
|
161
|
+
* that window failed because the database refused connections, not for its
|
|
162
|
+
* files (`local dev` retries it).
|
|
163
|
+
*/
|
|
164
|
+
export function snapshotInProgress(project: LocalProject, now = Date.now()): boolean {
|
|
165
|
+
const text = readText(lockPath(project))
|
|
166
|
+
if (!text) return false
|
|
167
|
+
const [pid, at] = text.trim().split(' ').map(Number)
|
|
168
|
+
return isAlive(pid) && now - (at ?? 0) < 5 * 60_000
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function sqlName(name: string): string {
|
|
172
|
+
if (!/^[a-z_][a-z0-9_]*$/.test(name)) throw new Error(`not a plain database name: ${name}`)
|
|
173
|
+
return name
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** One statement on the maintenance database, stopping at the first error. */
|
|
177
|
+
function psqlFor(project: LocalProject, exec: NonNullable<SnapshotDeps['exec']>) {
|
|
178
|
+
const container = containerName(project, 'postgres')
|
|
179
|
+
return (sql: string) => exec(container, ['psql', '-U', POSTGRES_USER, '-d', MAINTENANCE_DB, '-v', 'ON_ERROR_STOP=1', '-q', '-c', sql])
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const exists = (db: string) => `DO $$ BEGIN IF NOT EXISTS (SELECT FROM pg_database WHERE datname = '${sqlName(db)}') THEN RAISE EXCEPTION 'no database ${sqlName(db)}'; END IF; END $$`
|
|
183
|
+
const refuse = (db: string, allow: boolean) => `ALTER DATABASE ${sqlName(db)} ALLOW_CONNECTIONS ${allow}`
|
|
184
|
+
const terminate = (db: string) => `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${sqlName(db)}' AND pid <> pg_backend_pid()`
|
|
185
|
+
/** FILE_COPY: the copy is a file copy, not a WAL-logged one; the runtime runs with fsync off. */
|
|
186
|
+
const copy = (to: string, from: string) => `CREATE DATABASE ${sqlName(to)} TEMPLATE ${sqlName(from)} STRATEGY FILE_COPY`
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* The database copied while nobody holds it: connections refused, the open
|
|
190
|
+
* ones ended, the copy made (retried: an ended backend takes a moment to go).
|
|
191
|
+
*/
|
|
192
|
+
function copyWhileHeld(psql: (sql: string) => boolean, to: string, from: string, attempts = 3): boolean {
|
|
193
|
+
for (let i = 0; i < attempts; i++) {
|
|
194
|
+
psql(terminate(from))
|
|
195
|
+
if (psql(copy(to, from))) return true
|
|
196
|
+
}
|
|
197
|
+
return false
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* A moment for the pools to see their ended connections closed: a query sent
|
|
202
|
+
* on one before its close arrives fails (`CONNECTION_CLOSED`), so the first
|
|
203
|
+
* test right after a save would.
|
|
204
|
+
*/
|
|
205
|
+
function settle(deps: SnapshotDeps): Promise<void> {
|
|
206
|
+
const ms = deps.settleMs ?? 500
|
|
207
|
+
return ms > 0 ? Bun.sleep(ms) : Promise.resolve()
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function withLock<T>(project: LocalProject, run: () => Promise<T>): Promise<T> {
|
|
211
|
+
try {
|
|
212
|
+
writeFileSync(lockPath(project), `${process.pid} ${Date.now()}\n`)
|
|
213
|
+
} catch {
|
|
214
|
+
// The lock only tells `local dev` why an apply failed; the snapshot does not need it.
|
|
215
|
+
}
|
|
216
|
+
return run().finally(() => rmSync(lockPath(project), { force: true }))
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Take the baseline of the database as it is now, unless the stored one is of
|
|
221
|
+
* this build already (`force` takes it regardless).
|
|
222
|
+
*/
|
|
223
|
+
export async function saveBaseline(project: LocalProject, organizationId: string, deps: SnapshotDeps = {}, force = false): Promise<SnapshotOutcome> {
|
|
224
|
+
const exec = deps.exec ?? execInContainer
|
|
225
|
+
const now = deps.now ?? Date.now
|
|
226
|
+
const psql = psqlFor(project, exec)
|
|
227
|
+
const hash = buildHash(project, organizationId)
|
|
228
|
+
const stored = readBaseline(project, organizationId)
|
|
229
|
+
const started = now()
|
|
230
|
+
if (!force && stored?.buildHash === hash && psql(exists(BASELINE_DB))) {
|
|
231
|
+
return { ok: true, took: false, ms: 0, note: `the stored baseline is of this build (taken ${stored.savedAt})` }
|
|
232
|
+
}
|
|
233
|
+
return withLock(project, async () => {
|
|
234
|
+
// The copy before last is dropped first: a template is only ever this one build.
|
|
235
|
+
if (!psql(`DROP DATABASE IF EXISTS ${BASELINE_DB}`)) {
|
|
236
|
+
return { ok: false, took: false, ms: now() - started, note: `couldn’t drop the previous baseline (${BASELINE_DB}); see \`frontera local logs postgres\`` }
|
|
237
|
+
}
|
|
238
|
+
psql(refuse(POSTGRES_DB, false))
|
|
239
|
+
let copied = false
|
|
240
|
+
try {
|
|
241
|
+
copied = copyWhileHeld(psql, BASELINE_DB, POSTGRES_DB)
|
|
242
|
+
} finally {
|
|
243
|
+
psql(refuse(POSTGRES_DB, true))
|
|
244
|
+
}
|
|
245
|
+
const ms = now() - started
|
|
246
|
+
if (!copied) return { ok: false, took: false, ms, note: `couldn’t copy the database to ${BASELINE_DB}: something kept a connection open; see \`frontera local logs postgres\`` }
|
|
247
|
+
// Nothing connects to the baseline: it is only ever a template.
|
|
248
|
+
psql(refuse(BASELINE_DB, false))
|
|
249
|
+
await settle(deps)
|
|
250
|
+
const answering = deps.serviceAnswers ? await deps.serviceAnswers() : true
|
|
251
|
+
writeBaseline(project, {
|
|
252
|
+
version: 1,
|
|
253
|
+
organizationId,
|
|
254
|
+
database: POSTGRES_DB,
|
|
255
|
+
baseline: BASELINE_DB,
|
|
256
|
+
buildHash: hash,
|
|
257
|
+
savedAt: new Date(now()).toISOString(),
|
|
258
|
+
saveMs: ms,
|
|
259
|
+
applied: readText(hashesFile(project)),
|
|
260
|
+
build: readText(buildFile(project)),
|
|
261
|
+
apps: appDeployments(project.root),
|
|
262
|
+
})
|
|
263
|
+
const late = answering ? '' : '; the service isn’t answering since (see `frontera local logs service`)'
|
|
264
|
+
return { ok: true, took: true, ms, note: `took a baseline of the applied database (${ms} ms)${late}` }
|
|
265
|
+
})
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** How many targets the hash record says are applied differently from the baseline's copy. */
|
|
269
|
+
function changedSince(baseline: string | null, current: string | null): number {
|
|
270
|
+
const read = (text: string | null): Record<string, string> => {
|
|
271
|
+
try {
|
|
272
|
+
return (JSON.parse(text ?? '{}') as { hashes?: Record<string, string> }).hashes ?? {}
|
|
273
|
+
} catch {
|
|
274
|
+
return {}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
const was = read(baseline)
|
|
278
|
+
const now = read(current)
|
|
279
|
+
return new Set([...Object.keys(was), ...Object.keys(now)].filter((k) => was[k] !== now[k])).size
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Put the database back as the baseline holds it: connections refused and
|
|
284
|
+
* ended, the database dropped and made again from the baseline, the cache
|
|
285
|
+
* emptied. The CLI's record of what is applied is put back with it.
|
|
286
|
+
*/
|
|
287
|
+
export async function restoreBaseline(project: LocalProject, organizationId: string, deps: SnapshotDeps = {}, opts: { secrets?: RestoreSecrets } = {}): Promise<SnapshotOutcome> {
|
|
288
|
+
const exec = deps.exec ?? execInContainer
|
|
289
|
+
// A stand-in `exec` without a stand-in `query` carries nothing: tests never reach docker.
|
|
290
|
+
const query = deps.query ?? (deps.exec ? undefined : queryInContainer)
|
|
291
|
+
const keepSecrets = (opts.secrets ?? 'keep') === 'keep'
|
|
292
|
+
const now = deps.now ?? Date.now
|
|
293
|
+
const psql = psqlFor(project, exec)
|
|
294
|
+
const stored = readBaseline(project, organizationId)
|
|
295
|
+
if (!stored) return { ok: false, ms: 0, note: 'there is no baseline for this runtime yet: `frontera local snapshot save` takes one (`local check` takes one after it applies)' }
|
|
296
|
+
if (!psql(exists(BASELINE_DB))) return { ok: false, ms: 0, note: `the baseline database ${BASELINE_DB} is gone (a reset removes it): \`frontera local snapshot save\` takes a new one` }
|
|
297
|
+
const started = now()
|
|
298
|
+
return withLock(project, async () => {
|
|
299
|
+
// The secrets as they are now, read before the database goes: configuration, kept across the restore.
|
|
300
|
+
const pg = containerName(project, 'postgres')
|
|
301
|
+
const held = keepSecrets && query ? readSecrets(query, pg) : null
|
|
302
|
+
psql(refuse(POSTGRES_DB, false))
|
|
303
|
+
psql(terminate(POSTGRES_DB))
|
|
304
|
+
if (!psql(`DROP DATABASE ${POSTGRES_DB} WITH (FORCE)`)) {
|
|
305
|
+
psql(refuse(POSTGRES_DB, true))
|
|
306
|
+
return { ok: false, ms: now() - started, note: `couldn’t drop ${POSTGRES_DB} to restore it; the database is as the tests left it (see \`frontera local logs postgres\`)` }
|
|
307
|
+
}
|
|
308
|
+
// A new database takes connections: nothing is left refused, whichever way this ends.
|
|
309
|
+
if (!copyWhileHeld(psql, POSTGRES_DB, BASELINE_DB)) {
|
|
310
|
+
return { ok: false, ms: now() - started, note: `the database was dropped and couldn’t be made again from ${BASELINE_DB}: run \`frontera local snapshot restore\` to try again, or \`frontera local reset\`` }
|
|
311
|
+
}
|
|
312
|
+
// Before the cache is emptied and the service reads them: the secrets as they were.
|
|
313
|
+
const secrets: SnapshotOutcome['secrets'] = !keepSecrets
|
|
314
|
+
? { mode: 'baseline' }
|
|
315
|
+
: held && query
|
|
316
|
+
? { mode: 'keep', ...('problem' in held ? { ok: false, kept: 0, notKept: [], problem: held.problem } : carrySecrets(query, pg, held)) }
|
|
317
|
+
: undefined
|
|
318
|
+
// The service's cache holds rows of the database that was dropped.
|
|
319
|
+
exec(`frontera-${project.id}-cache`, ['redis-cli', 'FLUSHALL'])
|
|
320
|
+
await settle(deps)
|
|
321
|
+
// What is applied is what the baseline holds: anything applied after it is applied again.
|
|
322
|
+
const reapply = changedSince(stored.applied, readText(hashesFile(project)))
|
|
323
|
+
if (reapply) {
|
|
324
|
+
if (stored.applied !== null) writeFileAtomic(hashesFile(project), stored.applied, 0o600)
|
|
325
|
+
else rmSync(hashesFile(project), { force: true })
|
|
326
|
+
if (stored.build !== null) writeFileAtomic(buildFile(project), stored.build, 0o644)
|
|
327
|
+
}
|
|
328
|
+
// A Custom App deployed after the baseline: the restored database has no such deployment (or no such App).
|
|
329
|
+
// Its folder is put back as the baseline holds it, and marked, so status says so and `local dev` deploys it again.
|
|
330
|
+
const restoredAt = new Date(now()).toISOString()
|
|
331
|
+
const appsLost = appsLostByRestore(project.root, { atBaseline: stored.apps ?? null, savedAt: stored.savedAt })
|
|
332
|
+
markAppsLostByRestore(project.root, appsLost, restoredAt)
|
|
333
|
+
const answering = deps.serviceAnswers ? await deps.serviceAnswers() : true
|
|
334
|
+
const ms = now() - started
|
|
335
|
+
// Written last: a watching `local dev` reads it to take the record above.
|
|
336
|
+
writeBaseline(project, { ...stored, restoredAt, restoreMs: ms })
|
|
337
|
+
const kept = secretsWords(secrets)
|
|
338
|
+
const lostApps = appsLostWords(appsLost)
|
|
339
|
+
const tail = `${reapply ? `; ${reapply} item${reapply === 1 ? '' : 's'} applied after the baseline will apply again on the next \`frontera local dev\`` : ''}${lostApps ? `; ${lostApps}` : ''}${kept ? `; ${kept}` : ''}`
|
|
340
|
+
const withSecrets = secrets ? { secrets } : {}
|
|
341
|
+
const counted = { reapply: reapply + appsLost.length, ...(appsLost.length ? { appsLost } : {}) }
|
|
342
|
+
if (!answering) return { ok: false, ms, ...counted, ...withSecrets, note: `restored the database (${ms} ms), but the service isn’t answering since: see \`frontera local logs service\`${tail}` }
|
|
343
|
+
return { ok: true, ms, ...counted, ...withSecrets, note: `restored the database as applied (${ms} ms)${tail}` }
|
|
344
|
+
})
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** The record file's path relative to the project, which `local dev` watches for a restore. */
|
|
348
|
+
export const BASELINE_RECORD = join('.frontera', 'local', 'db-baseline.json')
|