@mnstry/atelier 0.2.0-alpha.1 → 0.2.0-alpha.11

package/src/projection/obsidian/publication/transport.mjs

@@ -0,0 +1,212 @@
+import { execFile, execFileSync } from 'node:child_process'
+import { readlinkSync } from 'node:fs'
+import { POLICY_SETTINGS_PATH, buildEvalCode, createInProcessHost, runInProcess, validatePayload } from './bridge-script.mjs'
+
+// Editor coordination adapter.
+//
+//   adapter.probe({ vaultRoot })  -> { state: 'coordinated' | 'absent' | 'uncoordinated', reason }
+//   adapter.inspect(payload)      -> reply          read-only, may retry
+//   adapter.collect(payload)      -> reply          read-only, may retry
+//   adapter.publish(payload)      -> reply          sent at most once
+//
+// A raw `call(payload)` sends one payload once and resolves with the parsed
+// reply. It throws TransportTimeout when the reply did not arrive in time and
+// any other error when the call could not be made. createEditorAdapter adds
+// the rules every transport must follow: calls are serialized, read-only calls
+// may retry, and a publish is never resent. When a publish reply is lost the
+// recorded outcome is re-read from the app; when none is recorded the caller
+// is told so and reconciles from the files themselves.
+
+export class TransportTimeout extends Error {
+  constructor(message) {
+    super(message)
+    this.name = 'TransportTimeout'
+    this.code = 'transport-timeout'
+  }
+}
+
+const READ_RETRIES = 2
+
+// `qualification`, when given, is the app qualification this adapter was
+// constructed under (`qualifyApp`). An adapter qualified while no app ran was
+// never shown a version, so it never coordinates with an app found running
+// later, whose version nobody checked: it is `uncoordinated` until an adapter
+// qualified with a checked version replaces it. Without a qualification the
+// adapter coordinates as before.
+export function createEditorAdapter({ call, processProbe, kind = 'custom', qualification }) {
+  if (typeof call !== 'function') throw new TypeError('createEditorAdapter needs a call function')
+  if (typeof processProbe !== 'function') throw new TypeError('createEditorAdapter needs a processProbe function')
+  const versionUnchecked = qualification !== undefined && qualification?.versionChecked !== true
+  const retries = []
+  let tail = Promise.resolve()
+  const serialized = (operation) => {
+    const next = tail.then(operation, operation)
+    tail = next.catch(() => {})
+    return next
+  }
+  const readOnly = async (payload) => {
+    for (let attempt = 0; ; attempt += 1) {
+      try { return await call(validatePayload(payload)) } catch (error) {
+        if (attempt >= READ_RETRIES || !(error instanceof TransportTimeout)) throw error
+        retries.push(payload.op)
+      }
+    }
+  }
+  return {
+    kind,
+    transportRetries: retries,
+    inspect: (payload) => serialized(() => readOnly({ ...payload, op: 'inspect' })),
+    collect: (payload) => serialized(() => readOnly({ ...payload, op: 'collect' })),
+    publish: (payload) => serialized(async () => {
+      validatePayload(payload)
+      if (payload.op !== 'publish') throw new TypeError('publish needs a publish payload')
+      try { return await call(payload) } catch (error) {
+        retries.push(error instanceof TransportTimeout ? 'publish-reply-lost' : 'publish-call-failed')
+        let record = null
+        try { record = await readOnly({ op: 'collect', vaultRoot: payload.vaultRoot, path: payload.path }) } catch { /* the app is not answering */ }
+        if (record && record.operationId === payload.operationId && typeof record.outcome === 'string') return { ...record, status: record.outcome, replyLost: true }
+        return { status: 'outcome-unknown', operationId: payload.operationId, wrote: null, replyLost: true, transportError: String(error.message || error).slice(0, 300) }
+      }
+    }),
+    // Path selection. `absent` is returned only when the process probe says,
+    // positively, that no Obsidian is running. A running or unknowable app
+    // that does not answer for this exact vault is `uncoordinated`, and so is
+    // any app while this adapter's qualification checked no version.
+    probe: ({ vaultRoot }) => serialized(async () => {
+      let processes
+      try { processes = await processProbe() } catch { processes = 'unknown' }
+      if (processes === 'absent') return { state: 'absent', reason: 'no Obsidian process is running' }
+      if (versionUnchecked) return { state: 'uncoordinated', reason: 'app-version-unchecked: an Obsidian process may be running, and this adapter was qualified while none ran, so its version was never checked' }
+      try {
+        const reply = await readOnly({ op: 'inspect', vaultRoot, path: POLICY_SETTINGS_PATH })
+        // `path-unsafe` for the probe path still proves the app answered for this vault; the settings unit reports the path.
+        if (['inspected', 'path-unsafe'].includes(reply.status) && reply.vaultBasePath === vaultRoot) return { state: 'coordinated', reason: 'the app answered for this vault' }
+        return { state: 'uncoordinated', reason: reply.status === 'vault-mismatch' ? 'the app answered for another vault' : `the app answered ${reply.status}` }
+      } catch (error) {
+        return { state: 'uncoordinated', reason: `an Obsidian process may be running and the bridge did not answer: ${String(error.message || error).slice(0, 200)}` }
+      }
+    }),
+  }
+}
+
+// ---------------------------------------------------------------------------
+// In-process transport: the same script, run here. Used for the path with no
+// app, and by tests that supply a host modelling an app.
+// ---------------------------------------------------------------------------
+
+export function createInProcessCall(host = createInProcessHost()) {
+  return async (payload) => runInProcess(payload, host)
+}
+
+export function createDirectAdapter({ crashSeam } = {}) {
+  return createEditorAdapter({ call: createInProcessCall(createInProcessHost({ crashSeam })), processProbe: () => 'absent', kind: 'direct' })
+}
+
+// ---------------------------------------------------------------------------
+// Obsidian CLI transport
+// ---------------------------------------------------------------------------
+
+export function defaultCliPath(platform = process.platform) {
+  if (platform === 'darwin') return '/Applications/Obsidian.app/Contents/MacOS/obsidian-cli'
+  return 'obsidian-cli'
+}
+
+// The CLI reaches the app through a socket under $HOME, so `env` selects the
+// app instance. It is passed explicitly and never edited here. A call that
+// outlives its timeout is killed with SIGKILL: the CLI ignores SIGTERM while
+// it waits on the app.
+export function createObsidianCliCall({ cliPath = defaultCliPath(), env = process.env, timeoutMs = 20000 } = {}) {
+  return (payload) => new Promise((resolve, reject) => {
+    execFile(cliPath, ['eval', `code=${buildEvalCode(payload)}`], { env, timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 16 * 1024 * 1024 }, (error, stdout, stderr) => {
+      if (error) return reject(error.killed ? new TransportTimeout(`CLI call timed out: ${payload.op}`) : new Error(`CLI call failed: ${String(error.message).slice(0, 300)}`))
+      const out = stdout || stderr || ''
+      const start = out.indexOf('=> ')
+      if (start < 0) return reject(new Error(`bridge returned no value: ${JSON.stringify(out.slice(0, 300))}`))
+      try { return resolve(JSON.parse(out.slice(start + 3))) } catch { return reject(new Error('bridge returned a value that is not JSON')) }
+    })
+  })
+}
+
+// The app is recognised by the executable a process runs, never by its
+// arguments: a path argument with an `obsidian` segment (this package's own
+// maintenance service, its private data root) is not the app. `ps -o comm=`
+// names the executable as it was started on macOS, usually its full path, and
+// by its command name of at most 15 characters on Linux.
+//
+//   macOS  `Obsidian` and the bundle's `Obsidian Helper…` processes, in any
+//          directory: a moved or renamed bundle still counts, and so does a
+//          process whose path ps cannot read and names by its short name.
+//   Linux  `obsidian`, the executable of the .deb, snap, AppImage and Flatpak
+//          builds, by name or, for a name that is not recognised, by the
+//          executable it resolves to. A distribution that runs the app under
+//          a system Electron shows only `electron`, and which app it hosts is
+//          not known.
+//
+// The command-line tool is a client of the app, not the app: counting it would
+// make every call this package makes through it look like a running app.
+const APP_EXECUTABLE = Object.freeze({ darwin: /^obsidian(?: helper\b.*)?$/i, linux: /^obsidian$/i })
+const ELECTRON_HOST = /^\.?electron(?:\d+|-wrap.*)?$/i
+// A `ps` that has not answered by then is killed, and the reading is unknown.
+const PS_TIMEOUT_MS = 5000
+// A process that exited since the table was read, or whose executable this
+// user may not read: another user's process, or one of this user's that hides
+// it from tracing (ssh-agent, a sandboxed renderer). Such a process is judged
+// by its name alone.
+const EXECUTABLE_UNREADABLE = new Set(['ENOENT', 'ESRCH', 'EACCES', 'EPERM'])
+
+const baseName = (text) => text.slice(text.lastIndexOf('/') + 1)
+const readProcessExecutable = (pid) => readlinkSync(`/proc/${pid}/exe`)
+
+// Linux names a process after the path it was started through, so an app
+// started through a differently named link or launcher shows another name.
+// The executable `/proc/<pid>/exe` resolves to does not change with the name.
+// Any failure other than the expected ones is unknown, and so is a table in
+// which no executable could be resolved at all: without `/proc`, nothing
+// below the names can be established.
+function resolveLinuxExecutables(rows, readExe) {
+  let resolved = 0
+  let unknown = false
+  for (const { pid } of rows) {
+    let target
+    try { target = String(readExe(pid)) } catch (error) {
+      if (!EXECUTABLE_UNREADABLE.has(error?.code)) unknown = true
+      continue
+    }
+    resolved += 1
+    const name = baseName(target.replace(/ \(deleted\)$/, ''))
+    if (APP_EXECUTABLE.linux.test(name)) return 'running'
+    if (ELECTRON_HOST.test(name)) unknown = true
+  }
+  return unknown || resolved === 0 ? 'unknown' : 'absent'
+}
+
+// Default process probe: 'absent' only when the process table was read and
+// holds no Obsidian. Limits: it sees this machine's processes as this user
+// can list them; it cannot see an app on another machine that reaches the
+// vault through a shared or synchronized folder, an app packaged under another
+// executable name, or an app that starts after the probe. Anything it cannot
+// establish is 'unknown', which the adapter treats as a running app: an empty
+// table, a failed or slow read, and on Linux a system Electron process or an
+// executable that cannot be resolved for an unexpected reason.
+export function defaultObsidianProcessProbe({ platform = process.platform, run = execFileSync, readExe = readProcessExecutable } = {}) {
+  if (!Object.hasOwn(APP_EXECUTABLE, platform)) return 'unknown'
+  const linux = platform === 'linux'
+  try {
+    const table = run('/bin/ps', ['-A', '-o', linux ? 'pid=,comm=' : 'comm='], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 16 * 1024 * 1024, timeout: PS_TIMEOUT_MS, killSignal: 'SIGKILL' })
+    if (typeof table !== 'string' || table.trim() === '') return 'unknown'
+    const rows = table.split('\n').map((line) => line.trim()).filter((line) => line !== '').map((line) => {
+      const match = linux ? /^(\d+)\s+(.*)$/.exec(line) : null
+      return match ? { pid: Number(match[1]), comm: match[2] } : { pid: null, comm: line }
+    })
+    if (rows.some((row) => APP_EXECUTABLE[platform].test(baseName(row.comm)))) return 'running'
+    if (!linux) return 'absent'
+    if (rows.some((row) => row.pid === null || ELECTRON_HOST.test(baseName(row.comm)))) return 'unknown'
+    return resolveLinuxExecutables(rows, readExe)
+  } catch {
+    return 'unknown'
+  }
+}
+
+export function createObsidianCliAdapter({ cliPath, env, timeoutMs, processProbe = () => defaultObsidianProcessProbe(), qualification } = {}) {
+  return createEditorAdapter({ call: createObsidianCliCall({ cliPath, env, timeoutMs }), processProbe, kind: 'obsidian-cli', qualification })
+}

package/src/projection/obsidian/recovery/index.mjs

@@ -0,0 +1,6 @@
+// Recovery state for Obsidian publication: the immutable store, the durable
+// journal, restart recovery and the late-writer re-check.
+export { EXCHANGE_CANDIDATE_NAME, LATE_EXCHANGE_CANDIDATE_NAME, PublicationRefusal, VAULT_LOCK_DIRECTORY, acquireVaultLock, createRecoveryStore, readFileDigest, sha256Digest } from './store.mjs'
+export { JOURNAL_SCHEMA, createJournal, journalDetail, listJournals, newJournalId } from './journal.mjs'
+export { classifyCandidateFile, namedCandidates, publishedSinceCommit, reconcileUnit, recoverPublications, retireStagedFile } from './restart.mjs'
+export { recheckDisplacedFiles } from './late-writer.mjs'

package/src/projection/obsidian/recovery/journal.mjs

@@ -0,0 +1,134 @@
+import { randomBytes } from 'node:crypto'
+import fs from 'node:fs'
+import path from 'node:path'
+import { isPendingPrivateWrite, publishPrivateFile } from '../../../project/durable-state.mjs'
+import { readRegularTextNoFollow } from '../../../project/private-state.mjs'
+import { OBSIDIAN_EXT_KEY, assertObsidianContract } from '../contracts.mjs'
+import { refuse } from './store.mjs'
+
+// Durable publication journal. The registered shape is one document; on disk
+// it is a header plus one immutable file per entry, each published atomically
+// under a name that is never overwritten, so an entry is either wholly present
+// or absent and history cannot be rewritten in place. The document is
+// assembled from those files and validated against the registered schema every
+// time it is read or extended.
+//
+// Entry steps follow the schema: `capture` (the write-ahead record binding a
+// note to its base, candidate, staged file and recovery path, written before
+// anything is exchanged), `conditional-update` (the exchange, exclusive
+// create or conditional removal and its outcome), `verify`, `manifest-commit`
+// and `restart`. Each entry file also carries the journal state that holds
+// after it.
+
+export const JOURNAL_SCHEMA = 'atelier-obsidian-publication-journal/v1'
+const ENTRY_FILE = /^\d{8}\.json$/
+const CLOSED_FILE = 'closed.json'
+
+const isoTime = (clock) => {
+  const value = clock()
+  const date = value instanceof Date ? value : new Date(value)
+  if (Number.isNaN(date.getTime())) throw new TypeError('the clock did not return a time')
+  return date.toISOString()
+}
+
+export function newJournalId(clock = () => new Date()) {
+  return `journal-${isoTime(clock).replace(/[-:.TZ]/g, '')}-${randomBytes(4).toString('hex')}`
+}
+
+function load(directory) {
+  const header = JSON.parse(readRegularTextNoFollow(path.join(directory, 'header.json')))
+  const entriesDir = path.join(directory, 'entries')
+  const names = fs.existsSync(entriesDir) ? fs.readdirSync(entriesDir).filter((name) => !isPendingPrivateWrite(name)).sort() : []
+  const records = names.map((name, index) => {
+    if (!ENTRY_FILE.test(name) || Number(name.slice(0, 8)) !== index) refuse('journal-corrupt', 'journal entries are not a contiguous sequence; preserve and inspect')
+    const record = JSON.parse(readRegularTextNoFollow(path.join(entriesDir, name)))
+    if (record.entry?.seq !== index) refuse('journal-corrupt', 'a journal entry does not carry its own sequence number')
+    return record
+  })
+  return { header, records }
+}
+
+function assemble({ header, records }) {
+  const restarts = records.filter((record) => record.entry.step === 'restart' && record.entry.ext?.[OBSIDIAN_EXT_KEY]?.phase === 'begin')
+  const document = {
+    ...header,
+    state: records.length > 0 ? records.at(-1).state : 'prepared',
+    entries: records.map((record) => record.entry),
+    restart: { count: restarts.length, ...(restarts.length > 0 ? { lastRecoveredAt: restarts.at(-1).entry.at } : {}) },
+  }
+  return assertObsidianContract('publication-journal', document)
+}
+
+function handle(store, directory, clock) {
+  // One writer at a time extends a journal (the publisher holds the view's
+  // lock), so the entries read once stay the entries on disk.
+  let cached = null
+  const current = () => (cached ??= load(directory))
+  const journal = {
+    directory,
+    document: () => assemble(current()),
+    // detail lands under the entry's ext container; the rest are schema fields.
+    append({ step, outcome, state, notePath, beforeDigest, afterDigest, recoveryRef, detail }) {
+      const loaded = current()
+      const entry = {
+        seq: loaded.records.length,
+        at: isoTime(clock),
+        step,
+        outcome,
+        ...(notePath === undefined ? {} : { notePath }),
+        ...(beforeDigest ? { beforeDigest } : {}),
+        ...(afterDigest ? { afterDigest } : {}),
+        ...(recoveryRef ? { recoveryRef } : {}),
+        ...(detail ? { ext: { [OBSIDIAN_EXT_KEY]: detail } } : {}),
+      }
+      const record = { entry, state: state ?? (loaded.records.at(-1)?.state ?? 'prepared') }
+      assemble({ header: loaded.header, records: [record] })
+      try {
+        publishPrivateFile(path.join(directory, 'entries', `${String(entry.seq).padStart(8, '0')}.json`), `${JSON.stringify(record, null, 2)}\n`)
+      } catch (error) {
+        cached = null
+        throw error
+      }
+      loaded.records.push(record)
+      return entry
+    },
+    // A journal that can no longer matter (committed, or superseded by a later
+    // commit) is closed so that later runs do not read it again. The flag file
+    // is advisory: a journal without one is simply read and judged again.
+    close() {
+      try { publishPrivateFile(path.join(directory, CLOSED_FILE), `${JSON.stringify({ state: assemble(current()).state })}\n`) } catch (error) { if (error.code !== 'EEXIST') throw error }
+    },
+  }
+  return journal
+}
+
+export function createJournal(store, { journalId, protocolId, expectedGeneration, targetGeneration, detail, clock = () => new Date() }) {
+  const directory = store.journalDir(journalId)
+  fs.mkdirSync(path.join(directory, 'entries'), { recursive: true, mode: 0o700 })
+  const header = {
+    schema: JOURNAL_SCHEMA,
+    contractVersion: '1.0.0',
+    journalId,
+    workspaceId: store.workspaceId,
+    scopeId: store.scopeId,
+    protocolId,
+    expectedGeneration,
+    targetGeneration,
+    ...(detail ? { ext: { [OBSIDIAN_EXT_KEY]: detail } } : {}),
+  }
+  assemble({ header, records: [] })
+  publishPrivateFile(path.join(directory, 'header.json'), `${JSON.stringify(header, null, 2)}\n`)
+  return handle(store, directory, clock)
+}
+
+// Journals of this view, oldest first. A directory without a header is a
+// journal that never began: nothing can refer to it.
+export function listJournals(store, { clock = () => new Date(), openOnly = false } = {}) {
+  if (!fs.existsSync(store.journalsRoot)) return []
+  return fs.readdirSync(store.journalsRoot).sort()
+    .filter((name) => fs.existsSync(path.join(store.journalsRoot, name, 'header.json')))
+    .filter((name) => !openOnly || !fs.existsSync(path.join(store.journalsRoot, name, CLOSED_FILE)))
+    .map((name) => handle(store, path.join(store.journalsRoot, name), clock))
+}
+
+export const journalDetail = (value) => value?.ext?.[OBSIDIAN_EXT_KEY] ?? {}

package/src/projection/obsidian/recovery/late-writer.mjs

@@ -0,0 +1,46 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { listJournals } from './journal.mjs'
+import { readFileBytes, sha256Digest } from './store.mjs'
+
+// A program that opened a note before it was exchanged still holds the
+// displaced file, and may write into it at any later time. Those bytes land in
+// the recovery area, not in the vault, so nothing shows them to the person
+// unless the displaced files are looked at again. This compares every
+// displaced file with the digest recorded when it was moved. A difference is
+// kept as an immutable object and reported as `late-writer-captured`.
+//
+// The check is meaningful only after a quiet period, and one pass does not end
+// the obligation: a holder can write later still. Callers repeat it.
+export function recheckDisplacedFiles({ store, journalIds, clock = () => new Date() } = {}) {
+  const ids = journalIds ?? listJournals(store).map((journal) => journal.document().journalId)
+  const findings = []
+  for (const journalId of ids) {
+    const receipts = store.listReceipts(journalId)
+    const known = new Set(receipts.filter((receipt) => receipt.role === 'late-writer').map((receipt) => `${receipt.unit}\u0000${receipt.observedDigest}`))
+    for (const receipt of receipts.filter((item) => item.role === 'displaced')) {
+      // A recovery name that is a hard link to the live note (an interrupted
+      // put-back) is the note itself: edits to it are not a late writer's.
+      const live = receipt.notePath ? fs.lstatSync(path.join(store.vaultRoot, receipt.notePath), { throwIfNoEntry: false }) : null
+      const held = fs.lstatSync(store.resolve(receipt.displacedRef), { throwIfNoEntry: false })
+      if (live && held && live.ino === held.ino && live.dev === held.dev) continue
+      let bytes
+      try { bytes = readFileBytes(store.resolve(receipt.displacedRef)) } catch (error) {
+        if (error.code !== 'ENOENT') throw error
+        findings.push({ code: 'displaced-file-missing', journalId, unit: receipt.unit, notePath: receipt.notePath, displacedRef: receipt.displacedRef })
+        continue
+      }
+      const observedDigest = sha256Digest(bytes)
+      if (observedDigest === receipt.digestAtMove) continue
+      const object = store.retainObject(bytes)
+      const finding = { code: 'late-writer-captured', journalId, unit: receipt.unit, notePath: receipt.notePath, displacedRef: receipt.displacedRef, digestAtMove: receipt.digestAtMove, observedDigest, objectRef: object.ref }
+      if (!known.has(`${receipt.unit}\u0000${observedDigest}`)) {
+        const value = clock()
+        store.writeReceipt(journalId, receipt.unit, { role: 'late-writer', notePath: receipt.notePath, displacedRef: receipt.displacedRef, digestAtMove: receipt.digestAtMove, observedDigest, objectRef: object.ref,
+          at: (value instanceof Date ? value : new Date(value)).toISOString() })
+      }
+      findings.push(finding)
+    }
+  }
+  return findings
+}