@graftland/cli 0.4.1 → 0.4.3

package/src/Manage.ts

@@ -12,6 +12,7 @@ import {
 	Record,
 	Schema,
 } from "effect"
+import { CliError } from "effect/cli"
 
 import * as Create from "@graftland/create"
 import * as PackageManager from "@graftland/package-manager"
@@ -20,6 +21,7 @@ import { parseDocument, type Document } from "yaml"
 
 import { CommandConfigError } from "./Commands"
 import { home } from "./internal/commandConfig"
+import { schemaFileName } from "./internal/schemaFileName"
 
 const report = (result: PackageManager.InstallResult) =>
 	Effect.logInfo(
@@ -52,6 +54,29 @@ const Spec = Schema.Union([
 ])
 const decodeSpecs = Schema.decodeEffect(Schema.Array(Spec))
 
+/**
+ * Regenerates the project's editor schema, but only when it has one, so that
+ * the plugins just added or removed are checked. Imports no plugin.
+ */
+const refreshSchema = Effect.fnUntraced(function* (directory: string) {
+	const fs = yield* FileSystem.FileSystem
+	const path = yield* Path.Path
+	const root = path.resolve(directory)
+	const file = path.join(root, schemaFileName)
+	if (!(yield* fs.exists(file))) return
+	const EditorSchema = yield* Effect.promise(
+		() => import("./internal/schemaFile"),
+	).pipe(Effect.uninterruptible)
+	yield* EditorSchema.write(root, file).pipe(
+		Effect.mapError((cause) =>
+			CliError.UserError.make({
+				cause,
+				userMessage: `Dependencies changed, but ${file} was not regenerated: ${cause.message}`,
+			}),
+		),
+	)
+})
+
 const install = Effect.fn("Manage.install")(function* (
 	directory: string,
 	changes: Readonly<Record<string, string | null>>,
@@ -64,6 +89,7 @@ const install = Effect.fn("Manage.install")(function* (
 	const manager = yield* PackageManager.make({ directory, userAgent })
 	const result = yield* manager.install(decoded)
 	yield* report(result)
+	yield* refreshSchema(directory)
 })
 
 export const add = Effect.fn("Manage.add")(function* (

package/src/Start.ts

@@ -30,7 +30,13 @@ import {
 } from "@graftland/loader"
 
 import { Application, ApplicationError, platform } from "./Application"
-import { layer as env, EnvFile, watchEnvFile } from "./internal/env"
+import {
+	layer as env,
+	EnvDir,
+	EnvFile,
+	watchEnvDir,
+	watchEnvFile,
+} from "./internal/env"
 import { layer as intercepts } from "./internal/intercepts"
 import { watchModules } from "./internal/watch"
 
@@ -44,6 +50,12 @@ export interface Options {
 	 * a variable. With `watch`, changes to it are read again.
 	 */
 	readonly envFile?: string | undefined
+	/**
+	 * A directory of secret files that `!env` reads when neither the process
+	 * environment nor `envFile` sets a variable. Each regular file is one
+	 * variable, named after the file. With `watch`, changes are read again.
+	 */
+	readonly envDir?: string | undefined
 }
 
 /**
@@ -138,7 +150,8 @@ export const run = (
 		yield* included.await
 		if (options.watch) {
 			const { reload } = yield* EnvFile
-			const envFile = options.envFile
+			const { reload: reloadDir } = yield* EnvDir
+			const { envFile, envDir } = options
 			const reloader = Plugin.make({
 				name: "reloader",
 				inject: [Loader.Loader, FileSystem.FileSystem, Path.Path],
@@ -152,6 +165,10 @@ export const run = (
 							yield* watchEnvFile(envFile, reload, loader.retry).pipe(
 								Effect.forkScoped,
 							)
+						if (envDir !== undefined)
+							yield* watchEnvDir(envDir, reloadDir, loader.retry).pipe(
+								Effect.forkScoped,
+							)
 					}),
 			})
 			yield* Graft.plugin(reloader)
@@ -171,7 +188,9 @@ export const run = (
 				Layer.mergeAll(
 					platform,
 					intercepts,
-					env(options.envFile).pipe(Layer.provide(NodeServices.layer)),
+					env({ file: options.envFile, dir: options.envDir }).pipe(
+						Layer.provide(NodeServices.layer),
+					),
 					Logger.layer([Logger.consolePretty()]),
 				),
 			),

package/src/bin.ts

@@ -12,6 +12,7 @@ import { Graft } from "@graftland/graft"
 import { version } from "../package.json" with { type: "json" }
 import { platform } from "./Application"
 import * as Host from "./internal/commandHost"
+import { schemaFileName } from "./internal/schemaFileName"
 
 const start = Command.make(
 	"start",
@@ -31,6 +32,12 @@ const start = Command.make(
 			),
 			Flag.optional,
 		),
+		envDir: Flag.String("env-dir").pipe(
+			Flag.withDescription(
+				"Read variables for !env from a directory with one file per variable, such as /run/secrets",
+			),
+			Flag.optional,
+		),
 	},
 	(options) =>
 		Effect.promise(() => import("./Start")).pipe(
@@ -39,6 +46,7 @@ const start = Command.make(
 				Start.run({
 					...options,
 					envFile: Option.getOrUndefined(options.envFile),
+					envDir: Option.getOrUndefined(options.envDir),
 				}),
 			),
 		),
@@ -90,6 +98,26 @@ const remove = Command.make(
 	),
 )
 
+const schema = Command.make(
+	"schema",
+	{
+		output: Flag.String("output").pipe(
+			Flag.withAlias("o"),
+			Flag.withDescription("Path of the schema file"),
+			Flag.withDefault(schemaFileName),
+		),
+	},
+	({ output }) =>
+		Effect.promise(() => import("./internal/schemaFile")).pipe(
+			Effect.uninterruptible,
+			Effect.flatMap((EditorSchema) => EditorSchema.write(".", output)),
+		),
+).pipe(
+	Command.withDescription(
+		"Write the JSON Schema of graft.yml, with the config of installed plugins",
+	),
+)
+
 const graft = Command.make("graft").pipe(
 	Command.withDescription("Run and manage Graft applications"),
 	Command.withSharedFlags({
@@ -106,7 +134,7 @@ const program = Effect.gen(function* () {
 			Effect.gen(function* () {
 				const context = yield* Layer.build(Graft.layer)
 				return yield* Host.run(
-					[start, create, add, remove],
+					[start, create, add, remove, schema],
 					graft,
 					version,
 					args,

package/src/internal/env.ts

@@ -1,8 +1,10 @@
 /**
  * Where `!env` values come from: the process environment, then an optional
- * `.env` file that watch mode reads again when it changes.
+ * `.env` file, then an optional directory of secret files. Watch mode reads
+ * the file and the directory again when they change.
  */
 import {
+	Array,
 	Cause,
 	ConfigProvider,
 	Context,
@@ -11,10 +13,13 @@ import {
 	Exit,
 	FileSystem,
 	Layer,
+	Option,
 	Path,
 	Queue,
+	Record,
 	Ref,
 	Stream,
+	String,
 	type PlatformError,
 } from "effect"
 
@@ -24,57 +29,156 @@ import { observeShutdown } from "./observeShutdown"
 // schema decides whether it is valid.
 const options = { preserveEmptyStrings: true } as const
 
-interface Loaded {
-	readonly target: string
-	readonly text: string
-	readonly provider: ConfigProvider.ConfigProvider
-}
+type Reload = Effect.Effect<boolean, PlatformError.PlatformError>
 
 export class EnvFile extends Context.Service<
 	EnvFile,
 	{
 		/** Reads the file again; reports whether its contents changed. */
-		readonly reload: Effect.Effect<boolean, PlatformError.PlatformError>
+		readonly reload: Reload
 	}
 >()("@graftland/cli/internal/env/EnvFile") {}
 
-/** Provides `ConfigProvider` and `EnvFile`; a missing file fails the layer. */
-export const layer = (file: string | undefined) =>
+export class EnvDir extends Context.Service<
+	EnvDir,
+	{
+		/** Reads the directory again; reports whether any value changed. */
+		readonly reload: Reload
+	}
+>()("@graftland/cli/internal/env/EnvDir") {}
+
+const unchanged = { reload: Effect.succeed(false) }
+
+/**
+ * A source that keeps the provider read from its latest snapshot. A failed
+ * read keeps the previous snapshot.
+ */
+const source = Effect.fnUntraced(function* <A>(
+	read: Effect.Effect<A, PlatformError.PlatformError>,
+	provider: (snapshot: A) => ConfigProvider.ConfigProvider,
+	same: (a: A, b: A) => boolean,
+) {
+	const initial = yield* read
+	const loaded = yield* Ref.make({
+		snapshot: initial,
+		provider: provider(initial),
+	})
+	return {
+		provider: ConfigProvider.make((path) =>
+			Effect.flatMap(Ref.get(loaded), ({ provider }) => provider.load(path)),
+		),
+		reload: Effect.gen(function* () {
+			const snapshot = yield* read
+			const previous = yield* Ref.getAndSet(loaded, {
+				snapshot,
+				provider: provider(snapshot),
+			})
+			return !same(snapshot, previous.snapshot)
+		}),
+	}
+})
+
+const readFile = Effect.fnUntraced(function* (file: string) {
+	const fs = yield* FileSystem.FileSystem
+	// Resolve the request on every read, including polling. Never pin a
+	// symlink to its initial target.
+	const target = yield* fs.realPath(file)
+	return yield* fs.readFileString(target)
+})
+
+const isNotFound = (error: PlatformError.PlatformError) =>
+	error.reason._tag === "NotFound"
+
+/**
+ * One variable per regular file, named after the file. Hidden entries are
+ * skipped, which also skips the bookkeeping links of Kubernetes volumes.
+ * Trailing line breaks are not part of the value.
+ */
+const readDir = Effect.fnUntraced(function* (directory: string) {
+	const fs = yield* FileSystem.FileSystem
+	const path = yield* Path.Path
+	const names = yield* fs.readDirectory(directory)
+	const values = yield* Effect.forEach(
+		Array.filter(names, (name) => !String.startsWith(".")(name)),
+		(name) => {
+			const file = path.join(directory, name)
+			return fs.stat(file).pipe(
+				Effect.flatMap((info) =>
+					info.type === "File"
+						? fs
+								.readFileString(file)
+								.pipe(
+									Effect.map((text) =>
+										Option.some([
+											name,
+											String.replace(/[\r\n]+$/, "")(text),
+										] as const),
+									),
+								)
+						: Effect.succeedNone,
+				),
+				// A file removed since the listing is no longer a variable.
+				Effect.catchIf(isNotFound, () => Effect.succeedNone),
+			)
+		},
+		{ concurrency: 16 },
+	)
+	return Record.fromEntries(Array.getSomes(values))
+})
+
+const sameValues = Record.makeEquivalence(String.Equivalence)
+
+/**
+ * Provides `ConfigProvider`, `EnvFile` and `EnvDir`. A missing file or
+ * directory fails the layer.
+ */
+export const layer = (sources: {
+	readonly file?: string | undefined
+	readonly dir?: string | undefined
+}) =>
 	Layer.unwrap(
 		Effect.gen(function* () {
-			const environment = ConfigProvider.fromEnv(options)
-			if (file === undefined)
-				return Layer.merge(
-					ConfigProvider.layer(environment),
-					Layer.succeed(EnvFile, { reload: Effect.succeed(false) }),
-				)
 			const fs = yield* FileSystem.FileSystem
-			// Resolve the request on every read, including polling. Never pin a
-			// symlink to its initial target. The provider and its baseline are
-			// committed together from the bytes actually read.
-			const read = Effect.gen(function* () {
-				const target = yield* fs.realPath(file)
-				const text = yield* fs.readFileString(target)
-				return {
-					target,
-					text,
-					provider: ConfigProvider.fromDotEnvContents(text, options),
-				}
-			})
-			const initial: Loaded = yield* read
-			const loaded = yield* Ref.make(initial)
-			const fromFile = ConfigProvider.make((path) =>
-				Effect.flatMap(Ref.get(loaded), ({ provider }) => provider.load(path)),
+			const path = yield* Path.Path
+			const provide = <A>(
+				read: Effect.Effect<
+					A,
+					PlatformError.PlatformError,
+					FileSystem.FileSystem | Path.Path
+				>,
+			) =>
+				read.pipe(
+					Effect.provideService(FileSystem.FileSystem, fs),
+					Effect.provideService(Path.Path, path),
+				)
+			const file =
+				sources.file === undefined
+					? undefined
+					: yield* source(
+							provide(readFile(sources.file)),
+							(text) => ConfigProvider.fromDotEnvContents(text, options),
+							(a, b) => a === b,
+						)
+			const dir =
+				sources.dir === undefined
+					? undefined
+					: yield* source(
+							provide(readDir(sources.dir)),
+							(values) => ConfigProvider.fromEnvRecord(values, options),
+							sameValues,
+						)
+			const provider = Array.reduce(
+				[file, dir],
+				ConfigProvider.fromEnv(options),
+				(first, next) =>
+					next === undefined
+						? first
+						: ConfigProvider.orElse(first, next.provider),
 			)
-			return Layer.merge(
-				ConfigProvider.layer(ConfigProvider.orElse(environment, fromFile)),
-				Layer.succeed(EnvFile, {
-					reload: Effect.gen(function* () {
-						const next = yield* read
-						const previous = yield* Ref.getAndSet(loaded, next)
-						return next.text !== previous.text
-					}),
-				}),
+			return Layer.mergeAll(
+				ConfigProvider.layer(provider),
+				Layer.succeed(EnvFile, file ?? unchanged),
+				Layer.succeed(EnvDir, dir ?? unchanged),
 			)
 		}),
 	)
@@ -98,11 +202,38 @@ const logStopped =
  */
 export const watchEnvFile = Effect.fnUntraced(function* <E, R>(
 	file: string,
-	reload: Effect.Effect<boolean, PlatformError.PlatformError>,
+	reload: Reload,
 	retry: Effect.Effect<void, E, R>,
 ) {
-	const fs = yield* FileSystem.FileSystem
 	const path = yield* Path.Path
+	const name = path.basename(file)
+	return yield* watchSource(
+		file,
+		path.dirname(file),
+		(changed) => path.basename(changed) === name,
+		reload,
+		retry,
+	)
+})
+
+/**
+ * Reloads the directory when anything in it changes and retries the loader's
+ * entries. A directory that cannot be read keeps its previous values.
+ */
+export const watchEnvDir = <E, R>(
+	directory: string,
+	reload: Reload,
+	retry: Effect.Effect<void, E, R>,
+) => watchSource(directory, directory, () => true, reload, retry)
+
+const watchSource = Effect.fnUntraced(function* <E, R>(
+	file: string,
+	directory: string,
+	matches: (path: string) => boolean,
+	reload: Reload,
+	retry: Effect.Effect<void, E, R>,
+) {
+	const fs = yield* FileSystem.FileSystem
 	const ready = yield* Deferred.make<void>()
 	// As in HMR, own the source scope inside the producer. Observe its
 	// complete Exit after cleanup, before merge can discard it.
@@ -112,13 +243,11 @@ export const watchEnvFile = Effect.fnUntraced(function* <E, R>(
 	>(
 		(queue) =>
 			fs
-				.watch(path.dirname(file), {
+				.watch(directory, {
 					onReady: () => Deferred.doneUnsafe(ready, Effect.void),
 				})
 				.pipe(
-					Stream.filter(
-						(event) => path.basename(event.path) === path.basename(file),
-					),
+					Stream.filter((event) => matches(event.path)),
 					Stream.runForEachArray((events) => Queue.offerAll(queue, events)),
 					Effect.scoped,
 					(effect) =>

package/src/internal/schemaFile.ts

@@ -0,0 +1,186 @@
+/** Composes the editor schema of a project's config file from its installed plugins. */
+import {
+	Array,
+	Effect,
+	FileSystem,
+	Option,
+	Path,
+	Predicate,
+	Record,
+	Schema,
+	type PlatformError,
+} from "effect"
+
+import { make } from "./configSchema"
+import * as Importer from "./importer"
+
+/** A dependency's `./schema` export could not be read. */
+export class SchemaRegistrationError extends Schema.TaggedError<
+	SchemaRegistrationError,
+	{ readonly brand: unique symbol }
+>()("SchemaRegistrationError", {
+	packageName: Schema.String,
+	detail: Schema.String,
+	cause: Schema.optionalKey(Schema.Unknown),
+}) {
+	override get message() {
+		return `${this.packageName}/schema: ${this.detail}`
+	}
+}
+
+const Dependencies = Schema.optionalKey(
+	Schema.Record(Schema.String, Schema.String),
+)
+const ProjectManifest = Schema.fromJsonString(
+	Schema.Struct({
+		dependencies: Dependencies,
+		devDependencies: Dependencies,
+		optionalDependencies: Dependencies,
+	}),
+)
+const InstalledManifest = Schema.fromJsonString(
+	Schema.Struct({
+		exports: Schema.optionalKey(Schema.Unknown),
+		dependencies: Dependencies,
+		peerDependencies: Dependencies,
+	}),
+)
+const Registrations = Schema.Array(
+	Schema.Struct({
+		names: Schema.Array(Schema.String),
+		schema: Schema.Record(Schema.String, Schema.Unknown),
+		required: Schema.optionalKey(Schema.Boolean),
+	}),
+)
+
+/**
+ * The manifest of an installed package, looked up as Node does: in
+ * `node_modules` of the directory and then of each parent, so hoisted
+ * workspace installs are found.
+ */
+const installedManifest = (
+	directory: string,
+	packageName: string,
+): Effect.Effect<
+	Option.Option<string>,
+	PlatformError.PlatformError,
+	FileSystem.FileSystem | Path.Path
+> =>
+	Effect.gen(function* () {
+		const fs = yield* FileSystem.FileSystem
+		const path = yield* Path.Path
+		const file = path.join(
+			directory,
+			"node_modules",
+			packageName,
+			"package.json",
+		)
+		if (yield* fs.exists(file)) return Option.some(file)
+		const parent = path.dirname(directory)
+		return parent === directory
+			? Option.none()
+			: yield* installedManifest(parent, packageName)
+	})
+
+/**
+ * Whether an installed dependency is a plugin package that exports `./schema`,
+ * read without importing it. Other packages, such as `effect`, may export a
+ * `./schema` of their own.
+ */
+const exportsSchema = Effect.fnUntraced(function* (
+	directory: string,
+	packageName: string,
+) {
+	const fs = yield* FileSystem.FileSystem
+	const file = yield* installedManifest(directory, packageName)
+	if (Option.isNone(file)) {
+		yield* Effect.logWarning(
+			`${packageName} is not installed; its config is left open`,
+		)
+		return false
+	}
+	const text = yield* fs.readFileString(file.value)
+	const manifest = yield* Schema.decodeEffect(InstalledManifest)(text)
+	return (
+		Predicate.hasProperty(manifest.exports, "./schema") &&
+		(Record.has(manifest.dependencies ?? {}, "@graftland/graft") ||
+			Record.has(manifest.peerDependencies ?? {}, "@graftland/graft"))
+	)
+})
+
+const load = Effect.fnUntraced(function* (
+	importer: (specifier: string) => Promise<unknown>,
+	packageName: string,
+) {
+	const failed = (detail: string) => (cause: unknown) =>
+		SchemaRegistrationError.make({ packageName, detail, cause })
+	const module = yield* Effect.tryPromise({
+		try: () => importer(`${packageName}/schema`),
+		catch: failed("cannot be imported"),
+	})
+	if (
+		!Predicate.hasProperty(module, "entries") ||
+		!Predicate.isFunction(module.entries)
+	)
+		return yield* SchemaRegistrationError.make({
+			packageName,
+			detail: "does not export an entries function",
+		})
+	const entries = module.entries
+	const value = yield* Effect.try({
+		try: () => entries(),
+		catch: failed("entries() threw"),
+	})
+	return yield* Schema.decodeUnknownEffect(Registrations)(value).pipe(
+		Effect.mapError(failed("entries() returned invalid registrations")),
+	)
+})
+
+/**
+ * The registrations exported as `./schema` by the dependencies in the
+ * project's package.json. Only these modules are imported, never the plugins.
+ */
+export const registrations = Effect.fn("EditorSchema.registrations")(function* (
+	directory: string,
+) {
+	const fs = yield* FileSystem.FileSystem
+	const path = yield* Path.Path
+	const text = yield* fs.readFileString(path.join(directory, "package.json"))
+	const manifest = yield* Schema.decodeEffect(ProjectManifest)(text)
+	const names = Array.dedupe(
+		Array.flatMap(
+			[
+				manifest.dependencies,
+				manifest.devDependencies,
+				manifest.optionalDependencies,
+			],
+			(dependencies) => Record.keys(dependencies ?? {}),
+		),
+	)
+	const plugins = yield* Effect.filter(names, (name) =>
+		exportsSchema(directory, name),
+	)
+	const importer = yield* Importer.make(directory)
+	const loaded = yield* Effect.forEach(plugins, (name) => load(importer, name))
+	return { plugins, entries: Array.flatten(loaded) }
+}, Effect.scoped)
+
+const encode = Schema.encodeEffect(
+	Schema.fromJsonString(Schema.Unknown, { space: "\t" }),
+)
+
+/** Writes the editor schema for the project in a directory. */
+export const write = Effect.fn("EditorSchema.write")(function* (
+	directory: string,
+	output: string,
+) {
+	const fs = yield* FileSystem.FileSystem
+	const { plugins, entries } = yield* registrations(directory)
+	const text = yield* encode(make(entries))
+	yield* fs.writeFileString(output, text + "\n")
+	yield* Effect.logInfo(
+		plugins.length === 0
+			? `Wrote ${output}; no dependency exports ./schema`
+			: `Wrote ${output} with ${Array.join(plugins, ", ")}`,
+	)
+})

package/src/internal/schemaFileName.ts

@@ -0,0 +1,2 @@
+/** The editor schema that `graft schema` writes, and that add, remove and create keep current. */
+export const schemaFileName = "graft.schema.json"