jackherd 1.15.0 → 1.15.2

package/src/watch/command.ts

@@ -32,6 +32,7 @@ import { existsSync, readFileSync } from 'node:fs'
 import { homedir } from 'node:os'
 import { basename, dirname, join, relative } from 'node:path'
 import { writePrivateFile } from '../../kit/hooks/hooks/shared/private-directory'
+import { sourcesWithPersonal } from '../config/personal-sources'
 import { readConfig, readResolvedConfig } from '../config/read-config'
 import type { JackherdConfig } from '../config/schema'
 import { resolveCoordinatorName } from '../context/coordinator-name'
@@ -182,9 +183,16 @@ import { withSingleReportNotice } from './single-report'
 import { readOrchestrateSnapshot, skillLineFor } from './skill-change'
 import { runsFromSource, sourceChangeSettled, watchSourceChanges } from './source-change'
 import { sourceDryRun } from './source-dry-run-command'
+import {
+  type LoadedSources,
+  nextLoadedSources,
+  readFileStamps,
+  readWatchedSources,
+  sourceFilePaths,
+} from './source-reload'
 import { runSource, SOURCE_LIMITS } from './source-run'
-import { recordWatchedSources } from './source-tickets'
-import { isSourceDue, type SourceRun, type SourceSettings, type SourceState, sourceEvents } from './sources'
+import { recordedSourceState } from './source-tickets'
+import { isSourceDue, type SourceRun, type SourceState, sourceEvents } from './sources'
 import { heartbeatStateOf, isHeartbeatNews, streamHeartbeatLine } from './stream-output'
 import {
   formatTaskParallelLine,
@@ -257,7 +265,10 @@ type Watch = {
   /** The switch `watch.parallelTasks.auto` and the paths that always ask, both from the main checkout's config. */
   parallelAuto: AutoSettings
   previews: PreviewsConfig
-  sources: SourceSettings[]
+  /** The sources as read at the start; the loop takes up later changes of their files (`Watching.sources`). */
+  sources: LoadedSources
+  /** The checkouts whose config the sources are read from, as `readResolvedConfig` takes them. */
+  configRoots: string[]
   texts: WatchTexts
   /** Where the orchestrate skill is installed, the project's copy first; the events name a reference inside the one found. */
   skillCopies: SkillCopy[]
@@ -305,6 +316,8 @@ type Watching = {
   failureReadings: FailureReading[]
   /** The last preview read that answered; recorded with every state like the feedback read. */
   previewRead: PreviewRead | null
+  /** The sources run now, read again whenever one of their files changes (`nextLoadedSources`). */
+  sources: LoadedSources
   /** The runs of the sources due at the last GitHub poll; recorded with every state like the feedback read. */
   sourceRuns: SourceRun[]
   /** The lanes' fills of the last read; a lane without an entry or with null has no value. */
@@ -381,7 +394,7 @@ const recordedOf = (watch: Watch, watching: Watching): WatchState => {
     unlinkedArtifacts: watching.unlinkedArtifacts,
     planMissing: watching.planMissing,
     approvals: watching.approvalRequests ? recordApprovals(watching.approvalRequests) : withReads.approvals,
-    sources: recordWatchedSources(withReads.sources, watching.sourceRuns, watch.sources, watching.githubReadAt),
+    ...recordedSourceState(withReads.sources, watching.sourceRuns, watching.sources.sources, watching.githubReadAt),
     appointments: watching.appointments
       ? recordAppointments(withReads.appointments, watching.appointments.table, watching.appointments.readAt)
       : withReads.appointments,
@@ -887,15 +900,26 @@ const recordPreviewRead = (watching: Watching, read: PreviewRead | null): WatchE
  * source stops the watcher. The run's start is recorded with the GitHub read's time, which is `now`.
  */
 const runDueSources = (watch: Watch, watching: Watching, now: number): Promise<SourceRun[]> => {
-  const due = watch.sources.filter((source) =>
+  const due = watching.sources.sources.filter((source) =>
     isSourceDue(watching.state.sources[source.name]?.ranAt, source.every, now),
   )
   return Promise.all(due.map((source) => runSource(source, watch.root, SOURCE_LIMITS)))
 }
 
-const recordSourceRuns = (watch: Watch, watching: Watching, runs: SourceRun[]): WatchEvent[] => {
+/**
+ * Takes up a change of the files the sources come from before the due ones run (`nextLoadedSources`): a source read
+ * anew has no `ranAt` and runs at once, its first run a baseline; an invalid file keeps the list and is said once.
+ */
+const reloadSources = (watch: Watch, watching: Watching) => {
+  const stamps = readFileStamps(Object.keys(watching.sources.stamps))
+  const next = nextLoadedSources(watching.sources, stamps, () => readWatchedSources(watch.configRoots, homedir()))
+  watching.sources = next.loaded
+  for (const line of next.lines) console.log(line)
+}
+
+const recordSourceRuns = (watching: Watching, runs: SourceRun[]): WatchEvent[] => {
   watching.sourceRuns = runs
-  return sourceEvents(watching.state.sources, runs, watch.sources)
+  return sourceEvents(watching.state.sources, runs, watching.sources.sources)
 }
 
 /** Starts a fetch of the base branch in the background when none runs; the loop does not wait for it. */
@@ -917,7 +941,8 @@ const syncPlanHeads = (watch: Watch, watching: Watching, pullRequests: readonly
 
 /**
  * Reads the coordinator's fill, GitHub, the feedback, the previews and the due sources, and stores them only once every read is complete: a SIGTERM
- * while a read is pending saves the state as it was, so no event is taken as reported.
+ * while a read is pending saves the state as it was, so no event is taken as reported. Before the sources run, a
+ * change of the files they come from is taken up (`reloadSources`), which prints its notes.
  */
 const pollGithubAndCoordinator = async (watch: Watch, watching: Watching, now: number): Promise<WatchEvent[]> => {
   syncBase(watch, watching)
@@ -925,6 +950,7 @@ const pollGithubAndCoordinator = async (watch: Watch, watching: Watching, now: n
   const fills = fillReadFor(watch, name, watching.agents, watching.lanes)
   const laneBranches = laneBranchesOf(watch, watching.lanes)
   const searched = Object.values({ ...watching.state.laneBranches, ...laneBranches })
+  reloadSources(watch, watching)
   const [github, sourceRuns] = await Promise.all([
     readGithub(watch.root, watch.deploy, watch.settings.baseBranch, searched),
     runDueSources(watch, watching, now),
@@ -940,7 +966,7 @@ const pollGithubAndCoordinator = async (watch: Watch, watching: Watching, now: n
     ...recordGithub(watching, github, laneBranches, now, failureReadings),
     ...recordFeedbackRead(watching, feedback?.read ?? null),
     ...recordPreviewRead(watching, previews),
-    ...recordSourceRuns(watch, watching, sourceRuns),
+    ...recordSourceRuns(watching, sourceRuns),
     ...recordCoordinator(watching, { name, read: fills.coordinator }),
     ...recordLaneFills(watching, fills.lanes),
     ...recordLaneModelReads(watching, laneModelReadingsOf(watch, watching, fills.models)),
@@ -1355,6 +1381,7 @@ const initialWatching = (
   feedbackRead: null,
   failureReadings: [],
   previewRead: null,
+  sources: watch.sources,
   sourceRuns: [],
   laneReadings: {},
   laneModelReadings: {},
@@ -1458,8 +1485,12 @@ const autoSettingsOf = (root: string, checkoutRoot: string, config: JackherdConf
 const watchOf = (options: WatchOptions): Watch => {
   const root = mainCheckout(process.cwd())
   const checkoutRoot = worktreeRoot(process.cwd())
-  const { config, notes } = readResolvedConfig([checkoutRoot, root])
-  for (const note of notes) console.log(note)
+  const configRoots = [checkoutRoot, root]
+  // Stamped before the read: a change in between is read again at the first poll, never missed.
+  const stamps = readFileStamps(sourceFilePaths(configRoots, homedir()))
+  const { config, notes } = readResolvedConfig(configRoots)
+  const merged = sourcesWithPersonal(config.watch.sources)
+  for (const note of [...notes, ...merged.notes]) console.log(note)
   const settings = plansSettingsOf(config)
   const picked = boardFor(options.boardPath, [root, worktreeRoot(process.cwd())], settings, () =>
     foundLanesOf(root, settings, () => agentNamesOf(listAgents())),
@@ -1478,7 +1509,8 @@ const watchOf = (options: WatchOptions): Watch => {
     taskSignalMinutes: config.watch.taskSignalMinutes,
     parallelAuto: autoSettingsOf(root, checkoutRoot, config),
     previews: config.watch.previews,
-    sources: config.watch.sources,
+    sources: { sources: merged.sources, stamps },
+    configRoots,
     texts: watchTexts(settings.responseLanguage),
     skillCopies: skillCopiesFor(root, homedir(), config.skillPrefix, DEFAULT_SKILL_PREFIX),
     herdr: insideHerdr(settings.adapter, process.env),

package/src/watch/plan-badges-read.ts

@@ -1,9 +1,10 @@
 import { dirname, join } from 'node:path'
+import { type HerdSourceSettings, herdSourceSettingsOf } from '../plans/settings'
 import { isPrivateLevels, type LocationSystem, readPrivateText, systemLocation } from '../shared/private-location'
 import {
   sourceTicketsOf,
-  type TicketSource,
   WATCH_HEALTH,
+  type WatchHealth,
   type WatchReading,
   watchBadgesOf,
   watchHealthOf,
@@ -22,21 +23,35 @@ const readState = (file: string): WatchState | null => {
   }
 }
 
+/** A reading with the sources it was drawn for: those the watcher wrote into its state, else the herd's own. */
+export type SourcedWatchReading = WatchReading & { sourceSettings: HerdSourceSettings }
+
+/**
+ * The sources a live watcher keeps in its state (`watchedSources`); `fallback`, the herd's config, for a watcher that
+ * stopped (its state may predate a config change) or a state written before it kept them.
+ */
+export const sourceSettingsOf = (
+  state: WatchState | null,
+  health: WatchHealth,
+  fallback: HerdSourceSettings,
+): HerdSourceSettings =>
+  health.kind === WATCH_HEALTH.live && state?.watchedSources ? herdSourceSettingsOf(state.watchedSources) : fallback
+
 /**
  * What the watcher of the repository at `root` gives the herd, read from its state file without a GitHub call:
- * its badges, the tickets of `ticketSources`, the status of the sources in `statusNames`, and its health, so the view can say why there are none. A missing or
- * damaged file shows no badges; the view goes on without them. Nothing is read unless both levels of the watcher
- * directory are private (`isPrivateLevels`), and only regular files, so a directory, a file or a FIFO someone else
- * planted neither feeds the view nor blocks it.
+ * its badges, the tickets of the ticket sources, the status of the other sources, and its health, so the view can say
+ * why there are none. The sources are those the state names, so one the watcher took up since the herd started shows
+ * too; `fallback` stands in for a state without them. A missing or damaged file shows no badges; the view goes on
+ * without them. Nothing is read unless both levels of the watcher directory are private (`isPrivateLevels`), and
+ * only regular files, so a directory, a file or a FIFO someone else planted neither feeds the view nor blocks it.
  */
 export const readWatchReading = (
   root: string,
   now: number,
-  ticketSources: readonly TicketSource[],
+  fallback: HerdSourceSettings,
   lookUp?: CommandLineOf,
   system: LocationSystem = systemLocation(),
-  statusNames: readonly string[] = [],
-): WatchReading => {
+): SourcedWatchReading => {
   const directory = watcherDirectoryOf(root, system)
   const isPrivate = isPrivateLevels(dirname(directory), directory, system)
   const state = isPrivate ? readState(join(directory, WATCHER_FILES.state)) : null
@@ -45,11 +60,14 @@ export const readWatchReading = (
     isPrivate && isWatcherRunning(join(directory, WATCHER_FILES.lock), lookUp),
     now,
   )
-  if (!state || health.kind !== WATCH_HEALTH.live) return { badges: {}, tickets: [], health }
+  const sourceSettings = sourceSettingsOf(state, health, fallback)
+  if (!state || health.kind !== WATCH_HEALTH.live) return { badges: {}, tickets: [], health, sourceSettings }
+  const statusNames = sourceSettings.herdSources.map(({ name }) => name)
   return {
     badges: watchBadgesOf(state, now),
-    tickets: sourceTicketsOf(state, ticketSources, now),
+    tickets: sourceTicketsOf(state, sourceSettings.ticketSources, now),
     health,
     ...(statusNames.length > 0 ? { sources: sourceStatusesOf(state, statusNames, now) } : {}),
+    sourceSettings,
   }
 }

package/src/watch/source-dry-run-command.ts

@@ -1,15 +1,17 @@
-import { readResolvedConfig } from '../config/read-config'
-import { type JackherdConfig, parseSourceDraft } from '../config/schema'
+import { homedir } from 'node:os'
+import { personalSourcesPath } from '../config/location'
+import { configuredSources, isNoteAbout } from '../config/personal-sources'
+import { parseSourceDraft } from '../config/schema'
 import { mainCheckout, worktreeRoot } from '../plans/git'
 import type { SourceDryRun } from './options'
 import { type DryRunTarget, dryRunExitCode, dryRunLines, dryRunTarget, emptyMapFields } from './source-dry-run'
-import { runSourceCommand, SOURCE_LIMITS, sourceRunOf } from './source-run'
+import { SOURCE_LIMITS, sourceAnswer, sourceRunOf } from './source-run'
 import type { SourceSettings } from './sources'
 
-const settingsOf = async (target: DryRunTarget, config: JackherdConfig): Promise<SourceSettings> => {
+const settingsOf = async (target: DryRunTarget, sources: readonly SourceSettings[]): Promise<SourceSettings> => {
   if (target.kind === 'name') {
-    const found = config.watch.sources.find(({ name }) => name === target.name)
-    if (!found) throw new Error(`no source ${target.name} in watch.sources`)
+    const found = sources.find(({ name }) => name === target.name)
+    if (!found) throw new Error(`no source ${target.name} in watch.sources or the personal sources file`)
     return found
   }
   const text = target.kind === 'stdin' ? await Bun.stdin.text() : await Bun.file(target.path).text()
@@ -17,19 +19,28 @@ const settingsOf = async (target: DryRunTarget, config: JackherdConfig): Promise
 }
 
 /**
- * `jh watch source <name|draft.json|-> --dry-run [--raw]`: runs one source once in the main checkout, as the
- * watcher would, and prints what it found and would report; `--raw` adds the command's output. Takes no lock and
+ * `jh watch source <name|draft.json|-> --dry-run [--raw]`: runs one source once in the main checkout (a personal one in `~/.jackherd`), as the
+ * watcher would, its `secrets` handed to the command alike, and prints what it found and would report; `--raw` adds
+ * the command's stdout, which by the recipe rules carries no secret value. Takes no lock and
  * neither reads nor writes the watcher's state, so a running watcher is not disturbed. Exits 1 when the run failed or a
  * `map` field is empty in every entry.
  */
 export const sourceDryRun = async ({ target, raw }: SourceDryRun): Promise<number> => {
   const root = mainCheckout(process.cwd())
-  const { config } = readResolvedConfig([worktreeRoot(process.cwd()), root])
-  const settings = await settingsOf(dryRunTarget(target), config)
-  const answer = await runSourceCommand(settings.command, root, SOURCE_LIMITS)
+  const home = homedir()
+  const merged = configuredSources([worktreeRoot(process.cwd()), root], home)
+  const settings = await settingsOf(dryRunTarget(target), merged.sources)
+  const answer = await sourceAnswer(settings, root, SOURCE_LIMITS, home)
   const run = sourceRunOf(settings, answer)
   const mapCheck = answer.ok ? emptyMapFields(answer.stdout, settings) : null
   const stdout = raw && answer.ok ? answer.stdout : undefined
-  console.log(dryRunLines(settings, run, { mapCheck, stdout }).join('\n'))
+  console.log(
+    dryRunLines(settings, run, {
+      mapCheck,
+      stdout,
+      personalPath: personalSourcesPath(home),
+      notes: merged.notes.filter((note) => isNoteAbout(note, settings.name)),
+    }).join('\n'),
+  )
   return dryRunExitCode(run, mapCheck)
 }

package/src/watch/source-dry-run.ts

@@ -3,6 +3,7 @@ import { sourceEventLines } from './lane-report'
 import {
   isIgnoredAuthor,
   listAt,
+  SOURCE_ORIGIN,
   type SourceItem,
   type SourceRun,
   type SourceSettings,
@@ -17,7 +18,14 @@ export type DryRunTarget = { kind: 'name'; name: string } | { kind: 'file'; path
 export type MapCheck = { total: number; empty: { field: string; path: string }[] }
 
 /** What one dry run shows besides the run: the map check (null when the output has no list) and the raw output. */
-export type DryRunExtras = { mapCheck: MapCheck | null; stdout?: string }
+export type DryRunExtras = {
+  mapCheck: MapCheck | null
+  stdout?: string
+  /** The personal sources file, named for a personal source. */
+  personalPath?: string
+  /** Notes of the merge that concern this source, such as the repository source that ran in place of a personal one. */
+  notes?: readonly string[]
+}
 
 const STDIN = '-'
 const LISTED_ENTRIES = 10
@@ -118,7 +126,16 @@ const exampleLines = (settings: SourceSettings, run: SourceRun): string[] => {
  */
 export const dryRunLines = (settings: SourceSettings, run: SourceRun, extras: DryRunExtras): string[] => {
   const head = `dry run source ${terminalSafe(settings.name)}: ${terminalSafe(JSON.stringify(settings.command))}`
-  const top = [head, ...(extras.stdout === undefined ? [] : rawLines(extras.stdout)), ...mapCheckLines(extras.mapCheck)]
+  const origin =
+    settings.origin === SOURCE_ORIGIN.personal ? [`origin: personal, ${terminalSafe(extras.personalPath ?? '')}`] : []
+  const notes = (extras.notes ?? []).map((note) => `note: ${terminalSafe(note)}`)
+  const top = [
+    head,
+    ...origin,
+    ...notes,
+    ...(extras.stdout === undefined ? [] : rawLines(extras.stdout)),
+    ...mapCheckLines(extras.mapCheck),
+  ]
   if (!run.result.ok) return [...top, ...eventsOf(settings, run).flatMap(sourceEventLines), FOOTER]
   return [...top, ...entryLines(run.result.items, settings), ...exampleLines(settings, run), FOOTER]
 }

package/src/watch/source-reload.ts

@@ -0,0 +1,100 @@
+import { existsSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+import { locateConfigFile, PROJECT_CONFIG_PATH, personalSourcesPath } from '../config/location'
+import { mergedSources, readPersonalSources } from '../config/personal-sources'
+import { readResolvedConfig } from '../config/read-config'
+import { errorText } from '../plans/error-text'
+import type { SourceSettings } from './sources'
+
+/** A file's change time in epoch milliseconds, null while it is missing; by path. */
+export type FileStamps = Record<string, number | null>
+
+/** The sources the watcher runs and the change times of the files they were read from. */
+export type LoadedSources = { sources: SourceSettings[]; stamps: FileStamps }
+
+/** A file the sources come from and how to read them; `read` throws for an invalid file. */
+export type SourceFileReader = { path: string; read: () => readonly SourceSettings[] }
+
+/** A file that could not be read, as the line the watcher prints. */
+type ReadFailure = { ok: false; line: string }
+
+/** A reading of both files: the merged sources with the notes of the skipped ones, or the line naming the file that failed. */
+export type SourceRead = { ok: true; sources: SourceSettings[]; notes: string[] } | ReadFailure
+
+/** What a line starts with when a changed file left the sources as they were. */
+export const NOT_RELOADED = 'sources not reloaded:'
+
+const sameStamps = (previous: FileStamps, current: FileStamps): boolean => {
+  const paths = new Set([...Object.keys(previous), ...Object.keys(current)])
+  return [...paths].every((path) => previous[path] === current[path])
+}
+
+/** The error of `path` on one line, without the `<path> is invalid:` the config readers put before the issues. */
+const reasonOf = (path: string, error: unknown): string =>
+  errorText(error).replace(`${path} is invalid:`, '').replace(/\s+/g, ' ').trim()
+
+const readOne = (reader: SourceFileReader): { ok: true; sources: readonly SourceSettings[] } | ReadFailure => {
+  try {
+    return { ok: true, sources: reader.read() }
+  } catch (error) {
+    return { ok: false, line: `${NOT_RELOADED} ${reader.path}: ${reasonOf(reader.path, error)}` }
+  }
+}
+
+/** Reads the repository's sources, then the personal ones, and merges them (`mergedSources`); the first file that fails is named. */
+export const readSourceFiles = (repo: SourceFileReader, personal: SourceFileReader): SourceRead => {
+  const repoRead = readOne(repo)
+  if (!repoRead.ok) return repoRead
+  const personalRead = readOne(personal)
+  if (!personalRead.ok) return personalRead
+  return { ok: true, ...mergedSources(repoRead.sources, personalRead.sources, personal.path) }
+}
+
+/**
+ * The sources after one cycle: with the stamps as they were, the same list without a read; after a change, the list
+ * `read` gives and its notes, or, when a changed file is invalid, the list as it was and the line naming the file.
+ * The new stamps are taken either way, so one change reports once and a source leaves the list only after a valid read.
+ */
+export const nextLoadedSources = (
+  previous: LoadedSources,
+  stamps: FileStamps,
+  read: () => SourceRead,
+): { loaded: LoadedSources; lines: string[] } => {
+  if (sameStamps(previous.stamps, stamps)) return { loaded: previous, lines: [] }
+  const result = read()
+  return result.ok
+    ? { loaded: { sources: result.sources, stamps }, lines: result.notes }
+    : { loaded: { sources: previous.sources, stamps }, lines: [result.line] }
+}
+
+/** The files the watcher's sources come from: the project config of each checkout in `roots`, once, then the personal file. */
+export const sourceFilePaths = (roots: readonly string[], home: string): string[] => [
+  ...new Set(roots.map((root) => join(root, PROJECT_CONFIG_PATH))),
+  personalSourcesPath(home),
+]
+
+const changeTimeOf = (path: string): number | null => {
+  try {
+    return statSync(path).mtimeMs
+  } catch {
+    return null
+  }
+}
+
+export const readFileStamps = (paths: readonly string[]): FileStamps =>
+  Object.fromEntries(paths.map((path) => [path, changeTimeOf(path)]))
+
+/**
+ * Reads the sources again: `watch.sources` of the config `readResolvedConfig` finds in `roots` (the rest of the config
+ * is not taken over), and those of the personal file of `home`. Unlike the start (`sourcesWithPersonal`), where an
+ * invalid personal file leaves the repository's sources alone, here it keeps the list the watcher runs: both keep
+ * what ran before, which at the start is no personal source.
+ */
+export const readWatchedSources = (roots: string[], home: string): SourceRead =>
+  readSourceFiles(
+    {
+      path: locateConfigFile(roots, home, existsSync) ?? join(roots[0] ?? home, PROJECT_CONFIG_PATH),
+      read: () => readResolvedConfig(roots, home).config.watch.sources,
+    },
+    { path: personalSourcesPath(home), read: (): SourceSettings[] => readPersonalSources(home) },
+  )

package/src/watch/source-run.ts

@@ -1,5 +1,16 @@
+import { homedir } from 'node:os'
+import { personalSourcesDirectory } from '../config/location'
+import { type Environment, sourceEnvironment } from '../secrets/environment'
+import { SECRET_LOOKUP, SECRET_TIMEOUT_MS, type SecretLookupFn, secretStore } from '../secrets/store'
 import { resolveSourceCommand } from './source-command'
-import { exitFailure, parseSourceOutput, SOURCE_FAILURE, type SourceRun, type SourceSettings } from './sources'
+import {
+  exitFailure,
+  parseSourceOutput,
+  SOURCE_FAILURE,
+  SOURCE_ORIGIN,
+  type SourceRun,
+  type SourceSettings,
+} from './sources'
 
 /** How long a source's command may run and how much it may print before it is stopped. */
 export type SourceLimits = { timeoutMs: number; maxBytes: number }
@@ -38,10 +49,11 @@ const readCapped = async (child: Child, maxBytes: number): Promise<string | null
 }
 
 /** Started in a process group of its own, so a timeout reaches what the command started too (`curl` behind `sh`). */
-const spawned = (command: readonly string[], cwd: string): Child | null => {
+const spawned = (command: readonly string[], cwd: string, env: Environment | undefined): Child | null => {
   try {
     return Bun.spawn(resolveSourceCommand(command, process.execPath), {
       cwd,
+      env,
       stdin: 'ignore',
       stdout: 'pipe',
       stderr: 'ignore',
@@ -62,7 +74,8 @@ const killGroup = (child: Child) => {
 }
 
 /**
- * Runs `command` as given, without a shell, in `cwd` with the user's environment; never rejects. stderr is not read,
+ * Runs `command` as given, without a shell, in `cwd` with `env` (the user's environment without it); never rejects.
+ * `env` reaches the command only: it is neither an argument nor printed. stderr is not read,
  * so nothing a failing tool prints there (a token, a URL with credentials) reaches the report. Past `timeoutMs` or
  * `maxBytes` the command's process group is killed, and the answer never waits past `timeoutMs` for a pipe a leftover
  * process keeps open; the group is also killed after a normal end, so a source leaves no process behind that stays in its group.
@@ -71,8 +84,9 @@ export const runSourceCommand = async (
   command: readonly string[],
   cwd: string,
   limits: SourceLimits,
+  env?: Environment,
 ): Promise<SourceAnswer> => {
-  const child = spawned(command, cwd)
+  const child = spawned(command, cwd, env)
   if (!child) return { ok: false, reason: SOURCE_FAILURE.notStarted }
   let timer: ReturnType<typeof setTimeout> | undefined
   const deadline = new Promise<typeof TIMED_OUT>((resolve) => {
@@ -92,6 +106,48 @@ export const runSourceCommand = async (
   }
 }
 
-/** One run of a source: its command, then its output over `items` and `map`. */
-export const runSource = async (settings: SourceSettings, cwd: string, limits: SourceLimits): Promise<SourceRun> =>
-  sourceRunOf(settings, await runSourceCommand(settings.command, cwd, limits))
+/** Where a source runs: a personal one in the personal directory under `home`, a repository one in `root`, the main checkout. */
+export const sourceDirectory = (settings: Pick<SourceSettings, 'origin'>, root: string, home: string): string =>
+  settings.origin === SOURCE_ORIGIN.personal ? personalSourcesDirectory(home) : root
+
+/**
+ * The keychain lookup a run uses unless it is handed another: bounded by the store's own limit and never past the
+ * run's, so a keychain that hangs costs each run at most that long.
+ */
+const keychainLookup = (limits: SourceLimits): SecretLookupFn =>
+  secretStore(Bun.secrets, Math.min(SECRET_TIMEOUT_MS, limits.timeoutMs)).get
+
+const noKeychain: SecretLookupFn = async () => ({ kind: SECRET_LOOKUP.absent })
+
+/**
+ * The lookup a source's secrets go through: the keychain for a personal source only. A repository source, which
+ * anyone with a commit can write, gets the watcher's environment and never a keychain value.
+ */
+export const secretLookupFor = (settings: Pick<SourceSettings, 'origin'>, keychain: SecretLookupFn): SecretLookupFn =>
+  settings.origin === SOURCE_ORIGIN.personal ? keychain : noKeychain
+
+/**
+ * The answer of a source's command in `sourceDirectory`, its `secrets` resolved first (`sourceEnvironment` over
+ * `secretLookupFor`, looked up afresh on each run and dropped after it). A secret that cannot be resolved fails the run
+ * before the command starts.
+ */
+export const sourceAnswer = async (
+  settings: SourceSettings,
+  root: string,
+  limits: SourceLimits,
+  home: string = homedir(),
+  lookup: SecretLookupFn = keychainLookup(limits),
+): Promise<SourceAnswer> => {
+  const environment = await sourceEnvironment(settings.secrets ?? [], secretLookupFor(settings, lookup), process.env)
+  if (!environment.ok) return environment
+  return runSourceCommand(settings.command, sourceDirectory(settings, root, home), limits, environment.environment)
+}
+
+/** One run of a source in `sourceDirectory`: its secrets and command (`sourceAnswer`), then its output over `items` and `map`. */
+export const runSource = async (
+  settings: SourceSettings,
+  root: string,
+  limits: SourceLimits,
+  home: string = homedir(),
+  lookup: SecretLookupFn = keychainLookup(limits),
+): Promise<SourceRun> => sourceRunOf(settings, await sourceAnswer(settings, root, limits, home, lookup))

package/src/watch/source-tickets.ts

@@ -15,6 +15,7 @@ import {
   storedValue,
   type TicketRecord,
 } from './sources'
+import { type WatchedSource, watchedSourceOf } from './watched-source'
 
 const ticketOf = (
   item: SourceItem,
@@ -117,3 +118,17 @@ export const recordWatchedSources = (
     settings,
     now,
   )
+
+/**
+ * The two parts of the watcher's state the sources write after `runs`: their records (`recordWatchedSources`) and
+ * what the herd draws them with (`watchedSources`), both from the sources the watcher runs now.
+ */
+export const recordedSourceState = (
+  previous: SourceState,
+  runs: readonly SourceRun[],
+  settings: readonly SourceSettings[],
+  now: number,
+): { sources: SourceState; watchedSources: WatchedSource[] } => ({
+  sources: recordWatchedSources(previous, runs, settings, now),
+  watchedSources: settings.map(watchedSourceOf),
+})

package/src/watch/sources.ts

@@ -1,4 +1,5 @@
 import { createHash } from 'node:crypto'
+import type { ToneName } from '../config/tones'
 import { terminalSafe } from '../plan-format/terminal-text'
 import { isRecord } from '../shared/is-record'
 import { previewUrlOf } from './previews'
@@ -35,6 +36,16 @@ export type SourceMap = {
   status?: SourcePath
 } & Record<string, SourcePath | undefined>
 
+/** Where a source is defined: the repository's config or the user's `~/.jackherd/sources.json`. */
+export const SOURCE_ORIGIN = { repo: 'repo', personal: 'personal' } as const
+
+export type SourceOrigin = (typeof SOURCE_ORIGIN)[keyof typeof SOURCE_ORIGIN]
+
+/** How the herd shows a source's entries: `entries` shows them open, in a block of their own. */
+export const SOURCE_SHOW = { entries: 'entries' } as const
+
+export type SourceShow = (typeof SOURCE_SHOW)[keyof typeof SOURCE_SHOW]
+
 /**
  * One `watch.sources` entry as the watcher uses it; `every` is in minutes, `items` the path of the list (the root
  * without it), `reportLimit` the most entries one report shows of it.
@@ -52,6 +63,16 @@ export type SourceSettings = {
   reportOn?: readonly string[]
   /** Its entries are tickets, which the state keeps for the herd's badges (`TicketRecord`). */
   ticket?: boolean
+  /** Names of the secrets the command gets as environment variables. */
+  secrets?: readonly string[]
+  /** `entries` shows its entries open, in a herd block of their own. */
+  show?: SourceShow
+  /** Where the source is defined; without it the repository's config. */
+  origin?: SourceOrigin
+  /** Shown before the source's rows in the herd; without it its name. */
+  label?: string
+  /** The tone of its mark in the herd while its newest report is new. */
+  tone?: ToneName
 }
 
 /**

package/src/watch/watch-events.ts

@@ -38,6 +38,7 @@ import {
   type UnlinkedArtifactEvent,
   type UnlinkedState,
 } from './unlinked-artifacts'
+import { isWatchedSource, type WatchedSource } from './watched-source'
 import { RUN_EVENT, RUN_STATUS, type WorkflowRun } from './workflow-runs'
 
 /** The event `updateAvailableLine` prints; it is no `WatchEvent`, the announcer logs it on its own. */
@@ -130,6 +131,11 @@ export type WatchState = {
   approvals: string[]
   /** Per configured source of `watch.sources`: the entries seen (`[id, fingerprint]`) and the start of a failure streak. */
   sources: SourceState
+  /**
+   * The sources the watcher runs, of the repository and the personal file, in its order and as the herd draws them;
+   * null in a state written before the watcher kept them, so the herd reads its own config.
+   */
+  watchedSources: WatchedSource[] | null
   /** Lane → change time of the report file last shown for it, null when it had none; a lane without an entry is unseen. */
   reportMtimes: Record<string, number | null>
   /** Lane → the questions of its report last shown or seen (`questionsFingerprint`); a lane without an entry is unseen. */
@@ -232,6 +238,7 @@ export const emptyWatchState = (): WatchState => ({
   planMissing: {},
   approvals: [],
   sources: emptySourceState(),
+  watchedSources: null,
   reportMtimes: {},
   reportQuestions: {},
   statusLines: null,
@@ -287,6 +294,8 @@ const isWatchState = (value: Record<string, unknown>): value is WatchState =>
   Array.isArray(value.approvals) &&
   value.approvals.every((id) => typeof id === 'string') &&
   isSourceState(value.sources) &&
+  (value.watchedSources === null ||
+    (Array.isArray(value.watchedSources) && value.watchedSources.every(isWatchedSource))) &&
   everyValue(value.reportMtimes, isSeq) &&
   everyValue(value.reportQuestions, (entry) => typeof entry === 'string') &&
   (value.statusLines === null ||