jackherd 1.15.1 → 1.15.2

package/src/watch/source-dry-run.ts

@@ -3,6 +3,7 @@ import { sourceEventLines } from './lane-report'
 import {
   isIgnoredAuthor,
   listAt,
+  SOURCE_ORIGIN,
   type SourceItem,
   type SourceRun,
   type SourceSettings,
@@ -17,7 +18,14 @@ export type DryRunTarget = { kind: 'name'; name: string } | { kind: 'file'; path
 export type MapCheck = { total: number; empty: { field: string; path: string }[] }
 
 /** What one dry run shows besides the run: the map check (null when the output has no list) and the raw output. */
-export type DryRunExtras = { mapCheck: MapCheck | null; stdout?: string }
+export type DryRunExtras = {
+  mapCheck: MapCheck | null
+  stdout?: string
+  /** The personal sources file, named for a personal source. */
+  personalPath?: string
+  /** Notes of the merge that concern this source, such as the repository source that ran in place of a personal one. */
+  notes?: readonly string[]
+}
 
 const STDIN = '-'
 const LISTED_ENTRIES = 10
@@ -118,7 +126,16 @@ const exampleLines = (settings: SourceSettings, run: SourceRun): string[] => {
  */
 export const dryRunLines = (settings: SourceSettings, run: SourceRun, extras: DryRunExtras): string[] => {
   const head = `dry run source ${terminalSafe(settings.name)}: ${terminalSafe(JSON.stringify(settings.command))}`
-  const top = [head, ...(extras.stdout === undefined ? [] : rawLines(extras.stdout)), ...mapCheckLines(extras.mapCheck)]
+  const origin =
+    settings.origin === SOURCE_ORIGIN.personal ? [`origin: personal, ${terminalSafe(extras.personalPath ?? '')}`] : []
+  const notes = (extras.notes ?? []).map((note) => `note: ${terminalSafe(note)}`)
+  const top = [
+    head,
+    ...origin,
+    ...notes,
+    ...(extras.stdout === undefined ? [] : rawLines(extras.stdout)),
+    ...mapCheckLines(extras.mapCheck),
+  ]
   if (!run.result.ok) return [...top, ...eventsOf(settings, run).flatMap(sourceEventLines), FOOTER]
   return [...top, ...entryLines(run.result.items, settings), ...exampleLines(settings, run), FOOTER]
 }

package/src/watch/source-reload.ts

@@ -0,0 +1,100 @@
+import { existsSync, statSync } from 'node:fs'
+import { join } from 'node:path'
+import { locateConfigFile, PROJECT_CONFIG_PATH, personalSourcesPath } from '../config/location'
+import { mergedSources, readPersonalSources } from '../config/personal-sources'
+import { readResolvedConfig } from '../config/read-config'
+import { errorText } from '../plans/error-text'
+import type { SourceSettings } from './sources'
+
+/** A file's change time in epoch milliseconds, null while it is missing; by path. */
+export type FileStamps = Record<string, number | null>
+
+/** The sources the watcher runs and the change times of the files they were read from. */
+export type LoadedSources = { sources: SourceSettings[]; stamps: FileStamps }
+
+/** A file the sources come from and how to read them; `read` throws for an invalid file. */
+export type SourceFileReader = { path: string; read: () => readonly SourceSettings[] }
+
+/** A file that could not be read, as the line the watcher prints. */
+type ReadFailure = { ok: false; line: string }
+
+/** A reading of both files: the merged sources with the notes of the skipped ones, or the line naming the file that failed. */
+export type SourceRead = { ok: true; sources: SourceSettings[]; notes: string[] } | ReadFailure
+
+/** What a line starts with when a changed file left the sources as they were. */
+export const NOT_RELOADED = 'sources not reloaded:'
+
+const sameStamps = (previous: FileStamps, current: FileStamps): boolean => {
+  const paths = new Set([...Object.keys(previous), ...Object.keys(current)])
+  return [...paths].every((path) => previous[path] === current[path])
+}
+
+/** The error of `path` on one line, without the `<path> is invalid:` the config readers put before the issues. */
+const reasonOf = (path: string, error: unknown): string =>
+  errorText(error).replace(`${path} is invalid:`, '').replace(/\s+/g, ' ').trim()
+
+const readOne = (reader: SourceFileReader): { ok: true; sources: readonly SourceSettings[] } | ReadFailure => {
+  try {
+    return { ok: true, sources: reader.read() }
+  } catch (error) {
+    return { ok: false, line: `${NOT_RELOADED} ${reader.path}: ${reasonOf(reader.path, error)}` }
+  }
+}
+
+/** Reads the repository's sources, then the personal ones, and merges them (`mergedSources`); the first file that fails is named. */
+export const readSourceFiles = (repo: SourceFileReader, personal: SourceFileReader): SourceRead => {
+  const repoRead = readOne(repo)
+  if (!repoRead.ok) return repoRead
+  const personalRead = readOne(personal)
+  if (!personalRead.ok) return personalRead
+  return { ok: true, ...mergedSources(repoRead.sources, personalRead.sources, personal.path) }
+}
+
+/**
+ * The sources after one cycle: with the stamps as they were, the same list without a read; after a change, the list
+ * `read` gives and its notes, or, when a changed file is invalid, the list as it was and the line naming the file.
+ * The new stamps are taken either way, so one change reports once and a source leaves the list only after a valid read.
+ */
+export const nextLoadedSources = (
+  previous: LoadedSources,
+  stamps: FileStamps,
+  read: () => SourceRead,
+): { loaded: LoadedSources; lines: string[] } => {
+  if (sameStamps(previous.stamps, stamps)) return { loaded: previous, lines: [] }
+  const result = read()
+  return result.ok
+    ? { loaded: { sources: result.sources, stamps }, lines: result.notes }
+    : { loaded: { sources: previous.sources, stamps }, lines: [result.line] }
+}
+
+/** The files the watcher's sources come from: the project config of each checkout in `roots`, once, then the personal file. */
+export const sourceFilePaths = (roots: readonly string[], home: string): string[] => [
+  ...new Set(roots.map((root) => join(root, PROJECT_CONFIG_PATH))),
+  personalSourcesPath(home),
+]
+
+const changeTimeOf = (path: string): number | null => {
+  try {
+    return statSync(path).mtimeMs
+  } catch {
+    return null
+  }
+}
+
+export const readFileStamps = (paths: readonly string[]): FileStamps =>
+  Object.fromEntries(paths.map((path) => [path, changeTimeOf(path)]))
+
+/**
+ * Reads the sources again: `watch.sources` of the config `readResolvedConfig` finds in `roots` (the rest of the config
+ * is not taken over), and those of the personal file of `home`. Unlike the start (`sourcesWithPersonal`), where an
+ * invalid personal file leaves the repository's sources alone, here it keeps the list the watcher runs: both keep
+ * what ran before, which at the start is no personal source.
+ */
+export const readWatchedSources = (roots: string[], home: string): SourceRead =>
+  readSourceFiles(
+    {
+      path: locateConfigFile(roots, home, existsSync) ?? join(roots[0] ?? home, PROJECT_CONFIG_PATH),
+      read: () => readResolvedConfig(roots, home).config.watch.sources,
+    },
+    { path: personalSourcesPath(home), read: (): SourceSettings[] => readPersonalSources(home) },
+  )

package/src/watch/source-run.ts

@@ -1,5 +1,16 @@
+import { homedir } from 'node:os'
+import { personalSourcesDirectory } from '../config/location'
+import { type Environment, sourceEnvironment } from '../secrets/environment'
+import { SECRET_LOOKUP, SECRET_TIMEOUT_MS, type SecretLookupFn, secretStore } from '../secrets/store'
 import { resolveSourceCommand } from './source-command'
-import { exitFailure, parseSourceOutput, SOURCE_FAILURE, type SourceRun, type SourceSettings } from './sources'
+import {
+  exitFailure,
+  parseSourceOutput,
+  SOURCE_FAILURE,
+  SOURCE_ORIGIN,
+  type SourceRun,
+  type SourceSettings,
+} from './sources'
 
 /** How long a source's command may run and how much it may print before it is stopped. */
 export type SourceLimits = { timeoutMs: number; maxBytes: number }
@@ -38,10 +49,11 @@ const readCapped = async (child: Child, maxBytes: number): Promise<string | null
 }
 
 /** Started in a process group of its own, so a timeout reaches what the command started too (`curl` behind `sh`). */
-const spawned = (command: readonly string[], cwd: string): Child | null => {
+const spawned = (command: readonly string[], cwd: string, env: Environment | undefined): Child | null => {
   try {
     return Bun.spawn(resolveSourceCommand(command, process.execPath), {
       cwd,
+      env,
       stdin: 'ignore',
       stdout: 'pipe',
       stderr: 'ignore',
@@ -62,7 +74,8 @@ const killGroup = (child: Child) => {
 }
 
 /**
- * Runs `command` as given, without a shell, in `cwd` with the user's environment; never rejects. stderr is not read,
+ * Runs `command` as given, without a shell, in `cwd` with `env` (the user's environment without it); never rejects.
+ * `env` reaches the command only: it is neither an argument nor printed. stderr is not read,
  * so nothing a failing tool prints there (a token, a URL with credentials) reaches the report. Past `timeoutMs` or
  * `maxBytes` the command's process group is killed, and the answer never waits past `timeoutMs` for a pipe a leftover
  * process keeps open; the group is also killed after a normal end, so a source leaves no process behind that stays in its group.
@@ -71,8 +84,9 @@ export const runSourceCommand = async (
   command: readonly string[],
   cwd: string,
   limits: SourceLimits,
+  env?: Environment,
 ): Promise<SourceAnswer> => {
-  const child = spawned(command, cwd)
+  const child = spawned(command, cwd, env)
   if (!child) return { ok: false, reason: SOURCE_FAILURE.notStarted }
   let timer: ReturnType<typeof setTimeout> | undefined
   const deadline = new Promise<typeof TIMED_OUT>((resolve) => {
@@ -92,6 +106,48 @@ export const runSourceCommand = async (
   }
 }
 
-/** One run of a source: its command, then its output over `items` and `map`. */
-export const runSource = async (settings: SourceSettings, cwd: string, limits: SourceLimits): Promise<SourceRun> =>
-  sourceRunOf(settings, await runSourceCommand(settings.command, cwd, limits))
+/** Where a source runs: a personal one in the personal directory under `home`, a repository one in `root`, the main checkout. */
+export const sourceDirectory = (settings: Pick<SourceSettings, 'origin'>, root: string, home: string): string =>
+  settings.origin === SOURCE_ORIGIN.personal ? personalSourcesDirectory(home) : root
+
+/**
+ * The keychain lookup a run uses unless it is handed another: bounded by the store's own limit and never past the
+ * run's, so a keychain that hangs costs each run at most that long.
+ */
+const keychainLookup = (limits: SourceLimits): SecretLookupFn =>
+  secretStore(Bun.secrets, Math.min(SECRET_TIMEOUT_MS, limits.timeoutMs)).get
+
+const noKeychain: SecretLookupFn = async () => ({ kind: SECRET_LOOKUP.absent })
+
+/**
+ * The lookup a source's secrets go through: the keychain for a personal source only. A repository source, which
+ * anyone with a commit can write, gets the watcher's environment and never a keychain value.
+ */
+export const secretLookupFor = (settings: Pick<SourceSettings, 'origin'>, keychain: SecretLookupFn): SecretLookupFn =>
+  settings.origin === SOURCE_ORIGIN.personal ? keychain : noKeychain
+
+/**
+ * The answer of a source's command in `sourceDirectory`, its `secrets` resolved first (`sourceEnvironment` over
+ * `secretLookupFor`, looked up afresh on each run and dropped after it). A secret that cannot be resolved fails the run
+ * before the command starts.
+ */
+export const sourceAnswer = async (
+  settings: SourceSettings,
+  root: string,
+  limits: SourceLimits,
+  home: string = homedir(),
+  lookup: SecretLookupFn = keychainLookup(limits),
+): Promise<SourceAnswer> => {
+  const environment = await sourceEnvironment(settings.secrets ?? [], secretLookupFor(settings, lookup), process.env)
+  if (!environment.ok) return environment
+  return runSourceCommand(settings.command, sourceDirectory(settings, root, home), limits, environment.environment)
+}
+
+/** One run of a source in `sourceDirectory`: its secrets and command (`sourceAnswer`), then its output over `items` and `map`. */
+export const runSource = async (
+  settings: SourceSettings,
+  root: string,
+  limits: SourceLimits,
+  home: string = homedir(),
+  lookup: SecretLookupFn = keychainLookup(limits),
+): Promise<SourceRun> => sourceRunOf(settings, await sourceAnswer(settings, root, limits, home, lookup))

package/src/watch/source-tickets.ts

@@ -15,6 +15,7 @@ import {
   storedValue,
   type TicketRecord,
 } from './sources'
+import { type WatchedSource, watchedSourceOf } from './watched-source'
 
 const ticketOf = (
   item: SourceItem,
@@ -117,3 +118,17 @@ export const recordWatchedSources = (
     settings,
     now,
   )
+
+/**
+ * The two parts of the watcher's state the sources write after `runs`: their records (`recordWatchedSources`) and
+ * what the herd draws them with (`watchedSources`), both from the sources the watcher runs now.
+ */
+export const recordedSourceState = (
+  previous: SourceState,
+  runs: readonly SourceRun[],
+  settings: readonly SourceSettings[],
+  now: number,
+): { sources: SourceState; watchedSources: WatchedSource[] } => ({
+  sources: recordWatchedSources(previous, runs, settings, now),
+  watchedSources: settings.map(watchedSourceOf),
+})

package/src/watch/sources.ts

@@ -1,4 +1,5 @@
 import { createHash } from 'node:crypto'
+import type { ToneName } from '../config/tones'
 import { terminalSafe } from '../plan-format/terminal-text'
 import { isRecord } from '../shared/is-record'
 import { previewUrlOf } from './previews'
@@ -35,6 +36,16 @@ export type SourceMap = {
   status?: SourcePath
 } & Record<string, SourcePath | undefined>
 
+/** Where a source is defined: the repository's config or the user's `~/.jackherd/sources.json`. */
+export const SOURCE_ORIGIN = { repo: 'repo', personal: 'personal' } as const
+
+export type SourceOrigin = (typeof SOURCE_ORIGIN)[keyof typeof SOURCE_ORIGIN]
+
+/** How the herd shows a source's entries: `entries` shows them open, in a block of their own. */
+export const SOURCE_SHOW = { entries: 'entries' } as const
+
+export type SourceShow = (typeof SOURCE_SHOW)[keyof typeof SOURCE_SHOW]
+
 /**
  * One `watch.sources` entry as the watcher uses it; `every` is in minutes, `items` the path of the list (the root
  * without it), `reportLimit` the most entries one report shows of it.
@@ -52,6 +63,16 @@ export type SourceSettings = {
   reportOn?: readonly string[]
   /** Its entries are tickets, which the state keeps for the herd's badges (`TicketRecord`). */
   ticket?: boolean
+  /** Names of the secrets the command gets as environment variables. */
+  secrets?: readonly string[]
+  /** `entries` shows its entries open, in a herd block of their own. */
+  show?: SourceShow
+  /** Where the source is defined; without it the repository's config. */
+  origin?: SourceOrigin
+  /** Shown before the source's rows in the herd; without it its name. */
+  label?: string
+  /** The tone of its mark in the herd while its newest report is new. */
+  tone?: ToneName
 }
 
 /**

package/src/watch/watch-events.ts

@@ -38,6 +38,7 @@ import {
   type UnlinkedArtifactEvent,
   type UnlinkedState,
 } from './unlinked-artifacts'
+import { isWatchedSource, type WatchedSource } from './watched-source'
 import { RUN_EVENT, RUN_STATUS, type WorkflowRun } from './workflow-runs'
 
 /** The event `updateAvailableLine` prints; it is no `WatchEvent`, the announcer logs it on its own. */
@@ -130,6 +131,11 @@ export type WatchState = {
   approvals: string[]
   /** Per configured source of `watch.sources`: the entries seen (`[id, fingerprint]`) and the start of a failure streak. */
   sources: SourceState
+  /**
+   * The sources the watcher runs, of the repository and the personal file, in its order and as the herd draws them;
+   * null in a state written before the watcher kept them, so the herd reads its own config.
+   */
+  watchedSources: WatchedSource[] | null
   /** Lane → change time of the report file last shown for it, null when it had none; a lane without an entry is unseen. */
   reportMtimes: Record<string, number | null>
   /** Lane → the questions of its report last shown or seen (`questionsFingerprint`); a lane without an entry is unseen. */
@@ -232,6 +238,7 @@ export const emptyWatchState = (): WatchState => ({
   planMissing: {},
   approvals: [],
   sources: emptySourceState(),
+  watchedSources: null,
   reportMtimes: {},
   reportQuestions: {},
   statusLines: null,
@@ -287,6 +294,8 @@ const isWatchState = (value: Record<string, unknown>): value is WatchState =>
   Array.isArray(value.approvals) &&
   value.approvals.every((id) => typeof id === 'string') &&
   isSourceState(value.sources) &&
+  (value.watchedSources === null ||
+    (Array.isArray(value.watchedSources) && value.watchedSources.every(isWatchedSource))) &&
   everyValue(value.reportMtimes, isSeq) &&
   everyValue(value.reportQuestions, (entry) => typeof entry === 'string') &&
   (value.statusLines === null ||

package/src/watch/watched-source.ts

@@ -0,0 +1,39 @@
+import { TONE_NAMES, type ToneName } from '../config/tones'
+import { isRecord } from '../shared/is-record'
+import { SOURCE_SHOW, type SourceSettings, type SourceShow } from './sources'
+
+/**
+ * What the herd needs of a source, as the watcher writes it into its state: its name, the `label` and `tone` it is
+ * drawn with, whether its entries are tickets, its lane and how it shows its entries; null where the config sets none.
+ */
+export type WatchedSource = {
+  name: string
+  label: string | null
+  tone: ToneName | null
+  ticket: boolean
+  lane: string | null
+  show: SourceShow | null
+}
+
+export const watchedSourceOf = ({ name, label, tone, ticket, lane, show }: SourceSettings): WatchedSource => ({
+  name,
+  label: label ?? null,
+  tone: tone ?? null,
+  ticket: ticket === true,
+  lane: lane ?? null,
+  show: show ?? null,
+})
+
+const isTextOrNull = (value: unknown): boolean => value === null || typeof value === 'string'
+
+const isOneOfOrNull = (values: readonly string[], value: unknown): boolean =>
+  value === null || (typeof value === 'string' && values.includes(value))
+
+export const isWatchedSource = (value: unknown): value is WatchedSource =>
+  isRecord(value) &&
+  typeof value.name === 'string' &&
+  isTextOrNull(value.label) &&
+  isOneOfOrNull(TONE_NAMES, value.tone) &&
+  typeof value.ticket === 'boolean' &&
+  isTextOrNull(value.lane) &&
+  isOneOfOrNull(Object.values(SOURCE_SHOW), value.show)

package/src/watch/watcher-lock.ts

@@ -85,6 +85,9 @@ export const repositoryKey = (root: string): string => {
 /** How long the herd trusts a command line `ps` gave for a process ID that is still alive. */
 const PS_CACHE_MS = 30_000
 
+/** Longest a `ps` call may take before it counts as no answer. */
+const PS_TIMEOUT_MS = 2000
+
 /** The command line from `/proc/<pid>/cmdline`, its arguments blank-separated; null where there is no `/proc` or no such process. */
 export const procCommandLine: CommandLineOf = (pid) => {
   try {
@@ -95,7 +98,11 @@ export const procCommandLine: CommandLineOf = (pid) => {
 }
 
 const psCommandLine: CommandLineOf = (pid) => {
-  const result = Bun.spawnSync(['ps', '-o', 'command=', '-p', String(pid)], { stdout: 'pipe', stderr: 'ignore' })
+  const result = Bun.spawnSync(['ps', '-o', 'command=', '-p', String(pid)], {
+    stdout: 'pipe',
+    stderr: 'ignore',
+    timeout: PS_TIMEOUT_MS,
+  })
   return result.exitCode === 0 ? result.stdout.toString().trim() || null : null
 }