@monster-orange/den 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -0
- package/README.md +1 -1
- package/docs/distributions.md +116 -1
- package/package.json +1 -1
- package/scripts/setup-secrets.ts +11 -0
- package/service/install.sh +14 -2
- package/src/shared/den-config.ts +69 -0
- package/src/shared/environment-manager/service-runner.ts +370 -6
- package/src/shared/manifest/schema.ts +15 -0
- package/src/shared/manifest/validate-layers.ts +40 -1
- package/src/shared/secrets/doppler.ts +10 -5
- package/src/shared/secrets/index.ts +1 -0
- package/src/shared/secrets/keys.ts +2 -0
- package/src/shared/secrets/secret-env.ts +149 -0
- package/src/shared/secrets/types.ts +10 -0
- package/src/shared/stacks/config.ts +6 -0
- package/src/shared/stacks/manifest-commands.ts +11 -1
- package/src/shared/stacks/runtime.ts +70 -2
- package/src/shared/stacks/types.ts +8 -0
- package/src/shared/types/index.ts +15 -0
- package/src/ui/cli/flag-parser.ts +2 -1
- package/src/ui/cli/index.ts +216 -34
- package/src/ui/cli/service-command.ts +131 -0
- package/src/ui/cli/stack-command.ts +55 -4
- package/src/ui/mcp/connect.ts +3 -1
- package/src/ui/mcp/server.ts +82 -1
- package/src/ui/server/index.ts +3 -0
- package/src/ui/server/lifecycle-exec.ts +23 -0
- package/src/ui/server/workspace-routes.ts +32 -0
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import { hasRuntime, repoKind } from "../repo-kind.js";
|
|
10
10
|
import { effectiveLocalCommands } from "../stacks/manifest-commands.js";
|
|
11
|
+
import { manifestStackName } from "../stacks/config.js";
|
|
11
12
|
import fs from "fs";
|
|
12
13
|
import path from "path";
|
|
13
14
|
import { readManifest } from "../manifest/read-manifest.js";
|
|
@@ -26,6 +27,15 @@ import {
|
|
|
26
27
|
shellQuote,
|
|
27
28
|
} from "../terminal/tabs.js";
|
|
28
29
|
import { readCatalog } from "./catalog.js";
|
|
30
|
+
import { getSecretsProvider } from "../secrets/index.js";
|
|
31
|
+
import {
|
|
32
|
+
assertSecretEnvSpec,
|
|
33
|
+
resolveSecretEnv,
|
|
34
|
+
secretEnvKeys,
|
|
35
|
+
type SecretEnvSpec,
|
|
36
|
+
} from "../secrets/secret-env.js";
|
|
37
|
+
import type { SecretsProvider } from "../secrets/types.js";
|
|
38
|
+
import { composeCommands } from "../stacks/manifest-commands.js";
|
|
29
39
|
import { buildServiceEnv } from "./service-env.js";
|
|
30
40
|
import {
|
|
31
41
|
devOpContainerNames,
|
|
@@ -40,6 +50,7 @@ import {
|
|
|
40
50
|
getActiveWorkspaces,
|
|
41
51
|
getFocusedWorkspaceName,
|
|
42
52
|
getWorkspaceByName,
|
|
53
|
+
parseAddress,
|
|
43
54
|
resolveServiceSource,
|
|
44
55
|
listWorkspaceServices,
|
|
45
56
|
} from "./workspaces.js";
|
|
@@ -49,7 +60,12 @@ import {
|
|
|
49
60
|
type ServiceHealthResult,
|
|
50
61
|
} from "./health.js";
|
|
51
62
|
import { EventEmitter } from "events";
|
|
52
|
-
import type {
|
|
63
|
+
import type {
|
|
64
|
+
CommandDef,
|
|
65
|
+
ResolvedService,
|
|
66
|
+
ServiceCatalog,
|
|
67
|
+
WorkspaceConfig,
|
|
68
|
+
} from "../types/index.js";
|
|
53
69
|
import type { ChildProcess } from "child_process";
|
|
54
70
|
import { getStateDir } from "../paths.js";
|
|
55
71
|
|
|
@@ -658,7 +674,95 @@ export function resolveService(
|
|
|
658
674
|
: getActiveWorkspace();
|
|
659
675
|
if (!ws) return null;
|
|
660
676
|
const catalog = readCatalog();
|
|
677
|
+
const listed = resolveListedService(address, ws, catalog);
|
|
678
|
+
if (listed || address.includes("/")) return listed;
|
|
679
|
+
// Not a repo or a listed address: a service key or a stack name, when it
|
|
680
|
+
// names exactly one service in the workspace.
|
|
681
|
+
const candidates = aliasCandidates(address, ws, catalog);
|
|
682
|
+
return candidates.length === 1
|
|
683
|
+
? resolveListedService(candidates[0], ws, catalog)
|
|
684
|
+
: null;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
type NamedWorkspace = { name: string; config: WorkspaceConfig };
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* The addresses `alias` could mean in `ws`: every service whose manifest key
|
|
691
|
+
* is `alias`, and the repo of every compose stack named `alias` (a stack
|
|
692
|
+
* stands for its repo; when that repo also has a service named `alias`, the
|
|
693
|
+
* service is the one match). Sorted; empty for a `repo/service` address and
|
|
694
|
+
* for a catalog repo's name.
|
|
695
|
+
*/
|
|
696
|
+
function aliasCandidates(
|
|
697
|
+
alias: string,
|
|
698
|
+
ws: NamedWorkspace,
|
|
699
|
+
catalog: ServiceCatalog,
|
|
700
|
+
): string[] {
|
|
701
|
+
// A repo name is never an alias, in the workspace or not.
|
|
702
|
+
if (alias.includes("/") || catalog[alias]) return [];
|
|
703
|
+
const services = new Set<string>();
|
|
704
|
+
const stackRepos = new Set<string>();
|
|
705
|
+
for (const s of listWorkspaceServices(ws.config, catalog, ws.name)) {
|
|
706
|
+
if (s.subsystem === alias) services.add(s.address);
|
|
707
|
+
const manifest = readManifest(s.repoPath);
|
|
708
|
+
if (manifest?.compose && manifestStackName(manifest, s.repoName) === alias)
|
|
709
|
+
stackRepos.add(s.repoName);
|
|
710
|
+
}
|
|
711
|
+
const withService = new Set(
|
|
712
|
+
[...services].map((a) => parseAddress(a, catalog).repoName),
|
|
713
|
+
);
|
|
714
|
+
return [
|
|
715
|
+
...services,
|
|
716
|
+
...[...stackRepos].filter((r) => !withService.has(r)),
|
|
717
|
+
].sort();
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
function namedWorkspace(workspaceName?: string): NamedWorkspace | null {
|
|
721
|
+
return workspaceName
|
|
722
|
+
? getWorkspaceByName(workspaceName)
|
|
723
|
+
: getActiveWorkspace();
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* What `alias` could address in the workspace (the focused one by default)
|
|
728
|
+
* when it is not a repo or a listed address: see resolveService.
|
|
729
|
+
*/
|
|
730
|
+
export function serviceAliasCandidates(
|
|
731
|
+
alias: string,
|
|
732
|
+
workspaceName?: string,
|
|
733
|
+
): string[] {
|
|
734
|
+
const ws = namedWorkspace(workspaceName);
|
|
735
|
+
if (!ws) return [];
|
|
736
|
+
const catalog = readCatalog();
|
|
737
|
+
if (resolveListedService(alias, ws, catalog)) return [];
|
|
738
|
+
return aliasCandidates(alias, ws, catalog);
|
|
739
|
+
}
|
|
661
740
|
|
|
741
|
+
/**
|
|
742
|
+
* The address to use for what someone typed: an unambiguous alias becomes
|
|
743
|
+
* the service's own address; anything else is returned as is. An ambiguous
|
|
744
|
+
* alias throws, naming the candidates.
|
|
745
|
+
*/
|
|
746
|
+
export function canonicalServiceAddress(
|
|
747
|
+
address: string,
|
|
748
|
+
workspaceName?: string,
|
|
749
|
+
): string {
|
|
750
|
+
const candidates = serviceAliasCandidates(address, workspaceName);
|
|
751
|
+
if (candidates.length > 1) {
|
|
752
|
+
const ws = namedWorkspace(workspaceName)?.name ?? "default";
|
|
753
|
+
throw new Error(
|
|
754
|
+
`"${address}" is ambiguous in workspace "${ws}": ${candidates.join(", ")}; name one of them`,
|
|
755
|
+
);
|
|
756
|
+
}
|
|
757
|
+
return candidates[0] ?? address;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/** resolveService for a repo name or a listed address (no aliases). */
|
|
761
|
+
function resolveListedService(
|
|
762
|
+
address: string,
|
|
763
|
+
ws: NamedWorkspace,
|
|
764
|
+
catalog: ServiceCatalog,
|
|
765
|
+
): ResolvedService | null {
|
|
662
766
|
let resolved = resolveServiceSource(address, ws.name, ws.config, catalog);
|
|
663
767
|
|
|
664
768
|
// Bare multi-service repo registered ONLY by subsystem (e.g. "shop/db" +
|
|
@@ -974,6 +1078,12 @@ export interface RunCommandOptions {
|
|
|
974
1078
|
* the detach window is invisible to fail-fast callers.
|
|
975
1079
|
*/
|
|
976
1080
|
onServiceExit?: (exitCode: number) => void;
|
|
1081
|
+
/**
|
|
1082
|
+
* R16: the service's resolved `secret_env` (see prepareServiceSecretEnv),
|
|
1083
|
+
* set only in the spawned process's env. An up/restart of a service that
|
|
1084
|
+
* declares secret_env is refused unless every key is here.
|
|
1085
|
+
*/
|
|
1086
|
+
secretEnv?: Record<string, string>;
|
|
977
1087
|
}
|
|
978
1088
|
|
|
979
1089
|
export interface RunCommandResult {
|
|
@@ -998,6 +1108,211 @@ export interface RunCommandResult {
|
|
|
998
1108
|
};
|
|
999
1109
|
}
|
|
1000
1110
|
|
|
1111
|
+
/**
|
|
1112
|
+
* A service with no commands of its own is started by its repo's up
|
|
1113
|
+
* (placement's repo-level rule). For `up` that is the repo's up (`address`
|
|
1114
|
+
* set); any other command would reach the repo's other services too, so it
|
|
1115
|
+
* points at the repo instead: a refusal, except that down succeeds (nothing
|
|
1116
|
+
* of the service's own runs, and a workspace Down sends every service).
|
|
1117
|
+
* Null when the rule does not apply.
|
|
1118
|
+
*/
|
|
1119
|
+
function repoStartedFallback(
|
|
1120
|
+
resolved: ResolvedService,
|
|
1121
|
+
command: string,
|
|
1122
|
+
workspace: string | undefined,
|
|
1123
|
+
): { address?: string; note: string } | null {
|
|
1124
|
+
const repoEntry = resolveService(resolved.repoName, workspace);
|
|
1125
|
+
if (!repoEntry || repoEntry.subsystem === resolved.subsystem) return null;
|
|
1126
|
+
const manifest = readManifest(resolved.repoPath);
|
|
1127
|
+
if (!manifest) return null;
|
|
1128
|
+
const repoCommands = effectiveLocalCommands(
|
|
1129
|
+
manifest,
|
|
1130
|
+
repoEntry.subsystem,
|
|
1131
|
+
resolved.repoName,
|
|
1132
|
+
workspace ?? "default",
|
|
1133
|
+
);
|
|
1134
|
+
if (!repoCommands["up"]) return null;
|
|
1135
|
+
if (command === "up")
|
|
1136
|
+
return {
|
|
1137
|
+
address: resolved.repoName,
|
|
1138
|
+
note: `${resolved.address} has no up of its own; running ${resolved.repoName}'s up, which starts it`,
|
|
1139
|
+
};
|
|
1140
|
+
return command === "down"
|
|
1141
|
+
? {
|
|
1142
|
+
note:
|
|
1143
|
+
`${resolved.address} has no down of its own; it is stopped by ${resolved.repoName}'s down. ` +
|
|
1144
|
+
`Run \`${product().cli} down ${resolved.repoName}\` (that stops every service it starts).`,
|
|
1145
|
+
}
|
|
1146
|
+
: {
|
|
1147
|
+
note:
|
|
1148
|
+
`${resolved.address} has no ${command} of its own; it is started by ${resolved.repoName}'s up. ` +
|
|
1149
|
+
`Run \`${product().cli} ${command} ${resolved.repoName}\` (that reaches every service it starts).`,
|
|
1150
|
+
};
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
// ── secret_env (R16) ─────────────────────────────────────────────────────────
|
|
1154
|
+
|
|
1155
|
+
/** Commands that start a service process, and so get its secret_env. */
|
|
1156
|
+
function startsService(command: string): boolean {
|
|
1157
|
+
return command === "up" || command === "restart";
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
/** A service's declared `services.<svc>.secret_env` (validated), or undefined. */
|
|
1161
|
+
function serviceSecretEnvSpec(
|
|
1162
|
+
resolved: ResolvedService,
|
|
1163
|
+
): SecretEnvSpec | undefined {
|
|
1164
|
+
const raw = readManifest(resolved.repoPath)?.services?.[resolved.subsystem]
|
|
1165
|
+
?.secret_env;
|
|
1166
|
+
return assertSecretEnvSpec(
|
|
1167
|
+
raw,
|
|
1168
|
+
`${resolved.address}: services.${resolved.subsystem}.secret_env`,
|
|
1169
|
+
);
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
/**
|
|
1173
|
+
* The address whose command a run of `command` on `address` actually spawns
|
|
1174
|
+
* (a service with no up of its own runs its repo's up), or null when none.
|
|
1175
|
+
*/
|
|
1176
|
+
function spawningAddress(
|
|
1177
|
+
address: string,
|
|
1178
|
+
command: string,
|
|
1179
|
+
workspace?: string,
|
|
1180
|
+
): string | null {
|
|
1181
|
+
const p = previewServiceCommand(address, command, workspace);
|
|
1182
|
+
return p.kind === "run" || p.kind === "via-repo" ? p.address : null;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
/**
|
|
1186
|
+
* The secret_env key NAMES a run of `command` on `address` would inject,
|
|
1187
|
+
* for a dry run or status: never a value, and the provider is never asked.
|
|
1188
|
+
* `service` is the service's own secret_env (set in its command's env);
|
|
1189
|
+
* `compose` is the repo's compose.secret_env, which Den's synthesized stack
|
|
1190
|
+
* command resolves for the docker compose process.
|
|
1191
|
+
*/
|
|
1192
|
+
export function previewServiceSecretEnv(
|
|
1193
|
+
address: string,
|
|
1194
|
+
command: string,
|
|
1195
|
+
workspace?: string,
|
|
1196
|
+
): { address: string; service: string[]; compose: string[] } | null {
|
|
1197
|
+
if (!startsService(command)) return null;
|
|
1198
|
+
const target = spawningAddress(address, command, workspace);
|
|
1199
|
+
const resolved = target ? resolveService(target, workspace) : null;
|
|
1200
|
+
if (!resolved) return null;
|
|
1201
|
+
const manifest = readManifest(resolved.repoPath);
|
|
1202
|
+
const service = Object.keys(
|
|
1203
|
+
manifest?.services?.[resolved.subsystem]?.secret_env ?? {},
|
|
1204
|
+
).sort();
|
|
1205
|
+
const declared = manifest?.services?.[resolved.subsystem]?.commands?.local;
|
|
1206
|
+
const synthesized =
|
|
1207
|
+
manifest?.compose && !declared?.[command]
|
|
1208
|
+
? composeCommands(
|
|
1209
|
+
manifest,
|
|
1210
|
+
resolved.subsystem,
|
|
1211
|
+
resolved.repoName,
|
|
1212
|
+
workspace ?? "default",
|
|
1213
|
+
)[command]
|
|
1214
|
+
: undefined;
|
|
1215
|
+
const compose = synthesized
|
|
1216
|
+
? Object.keys(manifest?.compose?.secret_env ?? {}).sort()
|
|
1217
|
+
: [];
|
|
1218
|
+
if (!service.length && !compose.length) return null;
|
|
1219
|
+
return { address: resolved.address, service, compose };
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
export type ServiceSecretEnvResult =
|
|
1223
|
+
| { ok: true; secretEnv: Record<string, string> }
|
|
1224
|
+
| { ok: false; message: string };
|
|
1225
|
+
|
|
1226
|
+
/**
|
|
1227
|
+
* Resolve the `secret_env` of the service a run of `command` on `address`
|
|
1228
|
+
* spawns, for runServiceCommand's `secretEnv`. All or nothing; the provider
|
|
1229
|
+
* is only asked when the service declares secret_env, and only for up and
|
|
1230
|
+
* restart. Messages name keys, refs and the provider, never values.
|
|
1231
|
+
*/
|
|
1232
|
+
export async function prepareServiceSecretEnv(
|
|
1233
|
+
address: string,
|
|
1234
|
+
command: string,
|
|
1235
|
+
workspace?: string,
|
|
1236
|
+
provider: () => SecretsProvider = getSecretsProvider,
|
|
1237
|
+
): Promise<ServiceSecretEnvResult> {
|
|
1238
|
+
if (!startsService(command)) return { ok: true, secretEnv: {} };
|
|
1239
|
+
const target = spawningAddress(address, command, workspace);
|
|
1240
|
+
const resolved = target ? resolveService(target, workspace) : null;
|
|
1241
|
+
if (!resolved) return { ok: true, secretEnv: {} };
|
|
1242
|
+
let spec: SecretEnvSpec | undefined;
|
|
1243
|
+
try {
|
|
1244
|
+
spec = serviceSecretEnvSpec(resolved);
|
|
1245
|
+
} catch (e) {
|
|
1246
|
+
return { ok: false, message: (e as Error).message };
|
|
1247
|
+
}
|
|
1248
|
+
if (!secretEnvKeys(spec).length) return { ok: true, secretEnv: {} };
|
|
1249
|
+
const r = await resolveSecretEnv(spec, provider(), resolved.address);
|
|
1250
|
+
return r.ok
|
|
1251
|
+
? { ok: true, secretEnv: r.env }
|
|
1252
|
+
: { ok: false, message: r.message };
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
/** What a run of `command` on `address` would do, without running it. */
|
|
1256
|
+
export type ServiceCommandPreview =
|
|
1257
|
+
| { kind: "run"; address: string }
|
|
1258
|
+
| { kind: "via-repo"; address: string; note: string }
|
|
1259
|
+
| { kind: "nothing"; address: string; note: string }
|
|
1260
|
+
| { kind: "error"; address: string; note: string };
|
|
1261
|
+
|
|
1262
|
+
/**
|
|
1263
|
+
* The dry-run twin of runServiceCommand's command lookup: the same
|
|
1264
|
+
* resolution, own-command, restart-synthesis and repo-started rules, so a
|
|
1265
|
+
* dry run says what a real run does.
|
|
1266
|
+
*/
|
|
1267
|
+
export function previewServiceCommand(
|
|
1268
|
+
address: string,
|
|
1269
|
+
command: string,
|
|
1270
|
+
workspace?: string,
|
|
1271
|
+
): ServiceCommandPreview {
|
|
1272
|
+
const resolved = resolveService(address, workspace);
|
|
1273
|
+
if (!resolved) {
|
|
1274
|
+
const candidates = serviceAliasCandidates(address, workspace);
|
|
1275
|
+
return {
|
|
1276
|
+
kind: "error",
|
|
1277
|
+
address,
|
|
1278
|
+
note:
|
|
1279
|
+
`service "${address}" not found in workspace "${workspace ?? "default"}"` +
|
|
1280
|
+
(candidates.length > 1
|
|
1281
|
+
? `; it is ambiguous: ${candidates.join(", ")}`
|
|
1282
|
+
: ""),
|
|
1283
|
+
};
|
|
1284
|
+
}
|
|
1285
|
+
const manifest = readManifest(resolved.repoPath);
|
|
1286
|
+
const own = manifest
|
|
1287
|
+
? effectiveLocalCommands(
|
|
1288
|
+
manifest,
|
|
1289
|
+
resolved.subsystem,
|
|
1290
|
+
resolved.repoName,
|
|
1291
|
+
workspace ?? "default",
|
|
1292
|
+
)
|
|
1293
|
+
: {};
|
|
1294
|
+
if (own[command] || (command === "restart" && own["down"] && own["up"]))
|
|
1295
|
+
return { kind: "run", address };
|
|
1296
|
+
const viaRepo = repoStartedFallback(resolved, command, workspace);
|
|
1297
|
+
if (viaRepo?.address)
|
|
1298
|
+
return { kind: "via-repo", address: viaRepo.address, note: viaRepo.note };
|
|
1299
|
+
if (viaRepo)
|
|
1300
|
+
return command === "down"
|
|
1301
|
+
? { kind: "nothing", address, note: viaRepo.note }
|
|
1302
|
+
: { kind: "error", address, note: viaRepo.note };
|
|
1303
|
+
if (command === "down")
|
|
1304
|
+
return {
|
|
1305
|
+
kind: "nothing",
|
|
1306
|
+
address,
|
|
1307
|
+
note: "no down command: nothing to stop",
|
|
1308
|
+
};
|
|
1309
|
+
return {
|
|
1310
|
+
kind: "error",
|
|
1311
|
+
address,
|
|
1312
|
+
note: `command "${command}" not found for service "${address}"`,
|
|
1313
|
+
};
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1001
1316
|
/**
|
|
1002
1317
|
* Resolve and execute a service command.
|
|
1003
1318
|
* Returns the spawned child process and whether it is a long-running (detachable) command.
|
|
@@ -1048,8 +1363,12 @@ export function runServiceCommand(
|
|
|
1048
1363
|
|
|
1049
1364
|
const resolved = resolveService(address, workspace);
|
|
1050
1365
|
if (!resolved) {
|
|
1366
|
+
const candidates = serviceAliasCandidates(address, workspace);
|
|
1051
1367
|
onLine(
|
|
1052
|
-
`Error: service "${address}" not found in catalog or workspace "${workspace}"
|
|
1368
|
+
`Error: service "${address}" not found in catalog or workspace "${workspace}"` +
|
|
1369
|
+
(candidates.length > 1
|
|
1370
|
+
? `; it is ambiguous: ${candidates.join(", ")}`
|
|
1371
|
+
: ""),
|
|
1053
1372
|
);
|
|
1054
1373
|
closeAsync(1);
|
|
1055
1374
|
return dummy();
|
|
@@ -1281,6 +1600,16 @@ export function runServiceCommand(
|
|
|
1281
1600
|
}
|
|
1282
1601
|
|
|
1283
1602
|
if (!cmdDef) {
|
|
1603
|
+
const viaRepo = repoStartedFallback(resolved, command, workspace);
|
|
1604
|
+
if (viaRepo?.address) {
|
|
1605
|
+
onLine(viaRepo.note);
|
|
1606
|
+
return runServiceCommand({ ...options, address: viaRepo.address });
|
|
1607
|
+
}
|
|
1608
|
+
if (viaRepo) {
|
|
1609
|
+
onLine(command === "down" ? viaRepo.note : `Error: ${viaRepo.note}`);
|
|
1610
|
+
closeAsync(command === "down" ? 0 : 1);
|
|
1611
|
+
return dummy();
|
|
1612
|
+
}
|
|
1284
1613
|
// Down is opctl-conventional: when no container is running AND the manifest
|
|
1285
1614
|
// declares no down command, there is simply nothing to stop — that's a
|
|
1286
1615
|
// successful down, not an error (keeps the ⋮ menu / workspace Down safe to
|
|
@@ -1304,6 +1633,41 @@ export function runServiceCommand(
|
|
|
1304
1633
|
onLine,
|
|
1305
1634
|
);
|
|
1306
1635
|
|
|
1636
|
+
// R16: a service that declares secret_env never starts without every key
|
|
1637
|
+
// (a caller that did not resolve them is refused, not run without them).
|
|
1638
|
+
let secretEnv: Record<string, string> = {};
|
|
1639
|
+
if (startsService(command)) {
|
|
1640
|
+
let declared: string[];
|
|
1641
|
+
try {
|
|
1642
|
+
declared = secretEnvKeys(serviceSecretEnvSpec(resolved));
|
|
1643
|
+
} catch (e) {
|
|
1644
|
+
onLine(`Error: ${(e as Error).message}`);
|
|
1645
|
+
closeAsync(1);
|
|
1646
|
+
return dummy();
|
|
1647
|
+
}
|
|
1648
|
+
const missing = declared.filter((k) => !options.secretEnv?.[k]);
|
|
1649
|
+
if (missing.length) {
|
|
1650
|
+
onLine(
|
|
1651
|
+
`Error: ${resolved.address} declares secret_env ${missing.join(", ")}, which were not resolved; not starting it`,
|
|
1652
|
+
);
|
|
1653
|
+
closeAsync(1);
|
|
1654
|
+
return dummy();
|
|
1655
|
+
}
|
|
1656
|
+
secretEnv = Object.fromEntries(
|
|
1657
|
+
declared.map((k) => [k, options.secretEnv?.[k] as string]),
|
|
1658
|
+
);
|
|
1659
|
+
}
|
|
1660
|
+
const hasSecrets = Object.keys(secretEnv).length > 0;
|
|
1661
|
+
// Secrets go only in the spawned process's env. A tab would need them in
|
|
1662
|
+
// its command line (and so its shell history), so such a start runs as a
|
|
1663
|
+
// daemon instead.
|
|
1664
|
+
const runEnv: NodeJS.ProcessEnv = { ...env, ...secretEnv };
|
|
1665
|
+
if (hasSecrets && runtime === "tab" && startsService(command))
|
|
1666
|
+
onLine(
|
|
1667
|
+
`${resolved.address} has secret_env: running as a daemon, not in a tab, to keep the values out of the tab's shell`,
|
|
1668
|
+
);
|
|
1669
|
+
const useTab = runtime === "tab" && !hasSecrets;
|
|
1670
|
+
|
|
1307
1671
|
// Manifest downs that are just `opctl container down <name>` must not be
|
|
1308
1672
|
// spawned raw: opctl re-elevates with `sudo -E` and fails on sudoers without
|
|
1309
1673
|
// SETENV even when there is nothing to stop — which is why files-api/api
|
|
@@ -1359,7 +1723,7 @@ export function runServiceCommand(
|
|
|
1359
1723
|
// HITL command — inherit the full TTY; onLine is not called.
|
|
1360
1724
|
const proc = spawnInteractive(
|
|
1361
1725
|
cmdString,
|
|
1362
|
-
{ ...process.env, ...
|
|
1726
|
+
{ ...process.env, ...runEnv },
|
|
1363
1727
|
onClose,
|
|
1364
1728
|
repoPath,
|
|
1365
1729
|
);
|
|
@@ -1409,11 +1773,11 @@ export function runServiceCommand(
|
|
|
1409
1773
|
}
|
|
1410
1774
|
};
|
|
1411
1775
|
|
|
1412
|
-
if (
|
|
1776
|
+
if (useTab && command === "up") {
|
|
1413
1777
|
return openCommandInTab(cmdString);
|
|
1414
1778
|
}
|
|
1415
1779
|
|
|
1416
|
-
if (
|
|
1780
|
+
if (useTab && command === "restart") {
|
|
1417
1781
|
closeExistingTab();
|
|
1418
1782
|
if (requestedCmdDef) return openCommandInTab(cmdString);
|
|
1419
1783
|
|
|
@@ -1531,7 +1895,7 @@ export function runServiceCommand(
|
|
|
1531
1895
|
const proc = spawnStreaming(
|
|
1532
1896
|
cmdString,
|
|
1533
1897
|
[],
|
|
1534
|
-
{ ...process.env, ...
|
|
1898
|
+
{ ...process.env, ...runEnv },
|
|
1535
1899
|
onLine,
|
|
1536
1900
|
wrappedOnClose,
|
|
1537
1901
|
repoPath,
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { z } from "zod";
|
|
11
11
|
import { journeysAppSchema } from "../../pillars/journeys/app-config.js";
|
|
12
|
+
import { SECRET_ENV_KEY_RE, secretEnvRefError } from "../secrets/secret-env.js";
|
|
12
13
|
|
|
13
14
|
export const CURRENT_SCHEMA_VERSION = 1;
|
|
14
15
|
export const KNOWN_VERSIONS = [1] as const;
|
|
@@ -69,6 +70,16 @@ const logSource = z.union([
|
|
|
69
70
|
}),
|
|
70
71
|
]);
|
|
71
72
|
|
|
73
|
+
// secret_env (R16): ENV_NAME → "<vault>/<item>/<field>", resolved through the
|
|
74
|
+
// secrets provider at up/restart and passed only in the process env.
|
|
75
|
+
const secretEnvConfig = z.record(
|
|
76
|
+
z.string().regex(SECRET_ENV_KEY_RE, "must be a valid env var name"),
|
|
77
|
+
z.string().superRefine((ref, ctx) => {
|
|
78
|
+
const err = secretEnvRefError(ref);
|
|
79
|
+
if (err) ctx.addIssue({ code: z.ZodIssueCode.custom, message: err });
|
|
80
|
+
}),
|
|
81
|
+
);
|
|
82
|
+
|
|
72
83
|
// ── Service management config ─────────────────────────────────────────────────
|
|
73
84
|
//
|
|
74
85
|
// Keys for commands/logs/health are environment names (local, dev, …).
|
|
@@ -105,6 +116,7 @@ const serviceTarget = z
|
|
|
105
116
|
const serviceManagementConfig = z.object({
|
|
106
117
|
runs: z.record(z.string(), z.enum(["local", "remote"])).optional(),
|
|
107
118
|
target: serviceTarget.optional(),
|
|
119
|
+
secret_env: secretEnvConfig.optional(),
|
|
108
120
|
depends_on: z.array(dependsOnEntry).optional(),
|
|
109
121
|
commands: z.record(z.string(), z.record(z.string(), commandDef)).optional(),
|
|
110
122
|
logs: z.record(z.string(), logSource).optional(),
|
|
@@ -280,6 +292,9 @@ const composeConfig = z.object({
|
|
|
280
292
|
z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])),
|
|
281
293
|
)
|
|
282
294
|
.optional(),
|
|
295
|
+
// Env vars resolved from the secrets provider into the docker compose
|
|
296
|
+
// process's env at up/restart. Never written to the override.
|
|
297
|
+
secret_env: secretEnvConfig.optional(),
|
|
283
298
|
});
|
|
284
299
|
|
|
285
300
|
// ── v1 manifest schema ────────────────────────────────────────────────────────
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
import * as fs from "fs";
|
|
18
|
+
import * as path from "path";
|
|
18
19
|
import { z } from "zod";
|
|
19
20
|
import type { ValidationError, ValidationResult } from "../types/index.js";
|
|
20
21
|
import {
|
|
@@ -24,11 +25,14 @@ import {
|
|
|
24
25
|
type ManifestMap,
|
|
25
26
|
} from "./merge-override.js";
|
|
26
27
|
import {
|
|
28
|
+
heldManifestPath,
|
|
27
29
|
loadManifest,
|
|
28
30
|
orphanOverrideMessage,
|
|
31
|
+
overrideSuffix,
|
|
29
32
|
parseManifestText,
|
|
30
33
|
type ManifestLayer,
|
|
31
34
|
} from "./read-manifest.js";
|
|
35
|
+
import { product } from "../product.js";
|
|
32
36
|
|
|
33
37
|
/** What layered validation needs from the schema module. */
|
|
34
38
|
export interface SchemaTools {
|
|
@@ -170,6 +174,31 @@ function readLayer(file: string): { value: unknown } | { error: string } {
|
|
|
170
174
|
}
|
|
171
175
|
}
|
|
172
176
|
|
|
177
|
+
/**
|
|
178
|
+
* A warning for a held manifest the repo's own manifest has made redundant:
|
|
179
|
+
* the repo's file always wins, so the held one is never read.
|
|
180
|
+
*/
|
|
181
|
+
function redundantHeldWarning(
|
|
182
|
+
heldFile: string,
|
|
183
|
+
overrideFile: string | null,
|
|
184
|
+
): ValidationError {
|
|
185
|
+
const yml = product().manifestYml;
|
|
186
|
+
const name = path.basename(heldFile);
|
|
187
|
+
const key = name.endsWith(yml) ? name.slice(0, -yml.length) : name;
|
|
188
|
+
const asOverride = path.join(
|
|
189
|
+
path.dirname(heldFile),
|
|
190
|
+
`${key}${overrideSuffix()}`,
|
|
191
|
+
);
|
|
192
|
+
return {
|
|
193
|
+
path: "",
|
|
194
|
+
file: heldFile,
|
|
195
|
+
message: `${heldFile} is not used: the repo now carries its own manifest, which always wins`,
|
|
196
|
+
suggestion: overrideFile
|
|
197
|
+
? `delete it; ${overrideFile} already layers onto the repo's manifest`
|
|
198
|
+
: `delete it, or turn it into an override (${asOverride}) that keeps only what this distribution changes`,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
173
202
|
/** Validate every manifest layer of `repoPath`, then the merged result. */
|
|
174
203
|
export function validateLayers(
|
|
175
204
|
tools: SchemaTools,
|
|
@@ -189,10 +218,20 @@ export function validateLayers(
|
|
|
189
218
|
}
|
|
190
219
|
|
|
191
220
|
const base = readLayer(baseFile);
|
|
192
|
-
|
|
221
|
+
let baseResult =
|
|
193
222
|
"error" in base
|
|
194
223
|
? failure(tools.version, baseFile, base.error)
|
|
195
224
|
: withFile(tools.validate(base.value), baseFile);
|
|
225
|
+
const redundantHeld =
|
|
226
|
+
baseLayer === "repo" ? heldManifestPath(repoPath) : null;
|
|
227
|
+
if (redundantHeld)
|
|
228
|
+
baseResult = {
|
|
229
|
+
...baseResult,
|
|
230
|
+
warnings: [
|
|
231
|
+
...baseResult.warnings,
|
|
232
|
+
redundantHeldWarning(redundantHeld, overrideFile),
|
|
233
|
+
],
|
|
234
|
+
};
|
|
196
235
|
layers.push({ layer: baseLayer, file: baseFile, result: baseResult });
|
|
197
236
|
|
|
198
237
|
if (overrideFile) {
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
*
|
|
5
5
|
* Project/config come from `secrets.doppler.{project,config}` in
|
|
6
6
|
* config/den.yml, else from the directory's `doppler setup` scope,
|
|
7
|
-
* else from DOPPLER_PROJECT / DOPPLER_CONFIG / DOPPLER_TOKEN.
|
|
7
|
+
* else from DOPPLER_PROJECT / DOPPLER_CONFIG / DOPPLER_TOKEN. A scoped ref
|
|
8
|
+
* (a `secret_env` value, `<project>/<config>/<key>`) names its own.
|
|
8
9
|
*/
|
|
9
10
|
import { execFile, spawn } from "child_process";
|
|
10
11
|
import { promisify } from "util";
|
|
@@ -29,8 +30,12 @@ export class DopplerProvider implements SecretsProvider {
|
|
|
29
30
|
this.timeout = opts.timeoutMs ?? 30_000;
|
|
30
31
|
}
|
|
31
32
|
|
|
32
|
-
/**
|
|
33
|
-
|
|
33
|
+
/**
|
|
34
|
+
* `--project/--config` flags: a scoped ref's own vault/item, else the
|
|
35
|
+
* configured ones (when set explicitly).
|
|
36
|
+
*/
|
|
37
|
+
scopeArgs(ref?: SecretRef): string[] {
|
|
38
|
+
if (ref?.scoped) return ["--project", ref.vault, "--config", ref.item];
|
|
34
39
|
const a: string[] = [];
|
|
35
40
|
if (this.opts.project) a.push("--project", this.opts.project);
|
|
36
41
|
if (this.opts.config) a.push("--config", this.opts.config);
|
|
@@ -56,7 +61,7 @@ export class DopplerProvider implements SecretsProvider {
|
|
|
56
61
|
const key = this.keyFor(ref);
|
|
57
62
|
const { stdout } = await execFileAsync(
|
|
58
63
|
"doppler",
|
|
59
|
-
["secrets", "get", key, "--plain", ...this.scopeArgs()],
|
|
64
|
+
["secrets", "get", key, "--plain", ...this.scopeArgs(ref)],
|
|
60
65
|
{ timeout: this.timeout },
|
|
61
66
|
);
|
|
62
67
|
return String(stdout).trim();
|
|
@@ -77,7 +82,7 @@ export class DopplerProvider implements SecretsProvider {
|
|
|
77
82
|
await new Promise<void>((resolve, reject) => {
|
|
78
83
|
const child = spawn(
|
|
79
84
|
"doppler",
|
|
80
|
-
["secrets", "set", key, "--silent", ...this.scopeArgs()],
|
|
85
|
+
["secrets", "set", key, "--silent", ...this.scopeArgs(ref)],
|
|
81
86
|
{ stdio: ["pipe", "ignore", "pipe"] },
|
|
82
87
|
);
|
|
83
88
|
let stderr = "";
|
|
@@ -26,6 +26,7 @@ export { flatKeyFor, toEnvName, refPath } from "./keys.js";
|
|
|
26
26
|
export { DopplerProvider } from "./doppler.js";
|
|
27
27
|
export { EnvProvider } from "./env.js";
|
|
28
28
|
export { OnePasswordProvider } from "./onepassword.js";
|
|
29
|
+
export * from "./secret-env.js";
|
|
29
30
|
|
|
30
31
|
export interface SecretsConfig {
|
|
31
32
|
provider?: SecretsProviderName;
|
|
@@ -16,6 +16,7 @@ export function refPath(ref: SecretRef): string {
|
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
18
|
* Resolve the flat key a key/value backend should use for `ref`:
|
|
19
|
+
* 0. a scoped ref's field, as is (it is fully qualified: no overrides);
|
|
19
20
|
* 1. `ref.key` when the caller pinned one;
|
|
20
21
|
* 2. the `secrets.keys` override map (`"vault/item/field": KEY`);
|
|
21
22
|
* 3. the field itself when it already looks like an env var (API keys);
|
|
@@ -25,6 +26,7 @@ export function flatKeyFor(
|
|
|
25
26
|
ref: SecretRef,
|
|
26
27
|
overrides: Record<string, string> = {},
|
|
27
28
|
): string {
|
|
29
|
+
if (ref.scoped) return ref.field;
|
|
28
30
|
if (ref.key) return ref.key;
|
|
29
31
|
const mapped = overrides[refPath(ref)];
|
|
30
32
|
if (mapped) return mapped;
|