@mnstry/atelier 0.2.0-alpha.12 → 0.2.0-alpha.14

package/src/projection/obsidian/proposals/adapter.mjs

@@ -6,7 +6,7 @@ import { AtelierDiagnosticError } from '../../../project/config.mjs'
 import { compareText, isoTime } from '../../../runtime/obsidian/documents.mjs'
 import { ObsidianMaintenanceRefusal } from '../../../runtime/obsidian/errors.mjs'
 import { readMachineSettings } from '../../../runtime/obsidian/machine-settings.mjs'
-import { DEFAULT_ELIGIBILITY, createProductionSeams } from '../../../runtime/obsidian/pipeline.mjs'
+import { createProductionSeams, eligibilityFor } from '../../../runtime/obsidian/pipeline.mjs'
 import { acquirePrivateGenerationLock, createAbandonmentProof } from '../../../runtime/obsidian/private-lock.mjs'
 import { OBSIDIAN_EXT_KEY, ObsidianContractRefusal, assertObsidianContract } from '../contracts.mjs'
 import { SOURCE_APPLY_PRIMITIVES } from '../edits/apply.mjs'
@@ -15,6 +15,8 @@ import { isIdentifier } from '../edits/object-identity.mjs'
 import { openObjectStore } from '../edits/object-store.mjs'
 import { decideApply } from '../edits/policy.mjs'
 import { applyEditLens } from '../edits/regions.mjs'
+import { createCooperativeBudget } from '../materialize/prepare-view.mjs'
+import { PERSONAL_VALIDATION_PENDING, personalOverlayRepoOf } from '../personal-workspace.mjs'
 import { PublicationRefusal } from '../recovery/store.mjs'
 import {
   PROPOSAL_BACKPRESSURE, PROPOSAL_LEDGER_LIMITS, classifyLedgerRead, classifyStoreRefusal, estimateEventLineBytes, isDue, isExhausted, nextAttemptAt, preflightAppend,
@@ -168,7 +170,7 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
   // two questions asked of the machine; `proveAbandoned` is the proof a held lock is passed over with.
   return function createProposalAdapter(options = {}) {
     const {
-      crash = () => {}, bounds: boundsInput = {}, limits = PROPOSAL_LEDGER_LIMITS, eligibility = DEFAULT_ELIGIBILITY,
+      crash = () => {}, bounds: boundsInput = {}, limits = PROPOSAL_LEDGER_LIMITS, eligibility: fixedEligibility = null,
       openStore = createProposalStore, objectStore = openObjectStore, isGitIgnored = SOURCE_APPLY_PRIMITIVES.isGitIgnored, proveAbandoned = createAbandonmentProof(), queueCrash = () => {},
     } = options
     const bounds = { ...PROPOSAL_BACKPRESSURE, ...boundsInput }
@@ -184,6 +186,12 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
     let observation = { key: null, run: null }
     // Edits whose operation is known not to be a proposal. A cache: losing it costs one read of an object.
     const notProposals = new Set()
+    // Set when an observation found the personal workspace's graph pending: the edits it could not look at are offered
+    // again on the next observation, not parked until a full reconciliation.
+    let observePendingAgain = false
+    // The observation under way, if any. An observation waits between two edits, and what it reads of this adapter
+    // (the flag above, the workspace it opens) is read when it begins: so the next one begins when this one has ended.
+    let observing = Promise.resolve()
 
     // -- the workspace of one call -------------------------------------------
 
@@ -198,12 +206,31 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
         if (!stores.has(scopeId)) stores.set(scopeId, seams.createRecoveryStore({ workspaceRoot, workspaceId, scopeId, repositoryRoots }))
         return stores.get(scopeId)
       }
-      // The canonical graph as it is now and the corpus profile of this machine, built once per call. Throws typed.
+      // The canonical graph as it is now and the corpus profile of this machine, built once per call with the notes the
+      // machine settings let into a view, as the engine builds it (eligibilityFor). Throws typed.
       let corpus = null
-      const currentCorpus = () => (corpus ??= {
-        graph: seams.buildGraph({ project, eligibility }),
-        profile: seams.profileFor({ project, workspaceId, audienceAllow: readMachineSettings({ workspaceRoot, workspaceId })?.audienceAllow ?? [] }),
-      })
+      // A pending answer is kept for the call too, so a call that looks at several edits builds at most once.
+      let pendingCorpus = null
+      const currentCorpus = () => {
+        if (corpus !== null) return corpus
+        if (pendingCorpus !== null) throw pendingCorpus
+        const machine = readMachineSettings({ workspaceRoot, workspaceId })
+        corpus = {
+          // The engine hands its deferral over: a personal workspace is then never composed on its event loop here.
+          graph: (() => {
+            try { return seams.buildGraph({ project, eligibility: fixedEligibility ?? eligibilityFor({ machine, project }), deferPersonalValidation: context.deferPersonalValidation === true }) } catch (error) {
+              if (error?.code === PERSONAL_VALIDATION_PENDING) pendingCorpus = error
+              throw error
+            }
+          })(),
+          profile: seams.profileFor({ project, workspaceId, audienceAllow: machine?.audienceAllow ?? [] }),
+        }
+        return corpus
+      }
+      // Whether the personal workspace's graph is pending on this call. Asked only where a graph would be built anyway.
+      const personalPending = () => {
+        try { currentCorpus(); return false } catch (error) { if (error?.code === PERSONAL_VALIDATION_PENDING) return true; if (isTyped(error)) return false; throw error }
+      }
       // true, false, or null when the canonical graph cannot be read now. The rule is the one source apply asks.
       const visible = options.isVisible ? (identity) => options.isVisible(identity, context) : ({ repoId, nodeId }) => {
         let built
@@ -223,6 +250,7 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
       return {
         ...context, recoveryOf, visible, clock, env: env ?? context.env ?? process.env,
         corpus: currentCorpus,
+        personalPending,
         objects: () => (objects ??= objectStore({ stateRoot: workspaceRoot, workspaceId, repositoryRoots, clock })),
         queue,
         managedRoots: () => [workspaceRoot, ...[...stores.values()].map((store) => store.vaultRoot)],
@@ -304,6 +332,8 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
         if (requireOpenEdit && editState !== 'open') return known ? refused(`edit-${editState}`) : outcome('skipped', null, { code: `edit-${editState}` })
         // The object store no longer says this is a proposed structural edit, so there is nothing to describe.
         if (item.observed === null) return refused('operation-not-proposed')
+        // A personal workspace whose graph is pending leaves the operation as it is: no attempt is spent on this tick.
+        if (workspace.deferPersonalValidation === true && personalOverlayRepoOf(workspace.project) !== null && workspace.personalPending()) return outcome('deferred', history?.head ?? null, { code: PERSONAL_VALIDATION_PENDING })
         const resolved = resolveProposalRoute({ project: workspace.project, workspaceId: workspace.workspaceId, identity: item, sourcePath: item.sourcePath, managedRoots: workspace.managedRoots(), isVisible: workspace.visible, isGitIgnored, env: env ?? workspace.env ?? process.env })
         if (!resolved.ok && !resolved.transient) return refused(resolved.code)
         // Recorded before anything is asked of the store.
@@ -405,10 +435,13 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
 
     // One repository: its lock, then each of its operations in turn. What goes wrong with one repository, or with one
     // operation, is reported by its code and stops nothing else.
-    async function settleRepository(workspace, repoId, works, report) {
+    // `budget`, when a tick hands one over, gives the event loop a turn between two operations: each operation is
+    // settled without interruption, and the lock of the repository is held from the first to the last.
+    async function settleRepository(workspace, repoId, works, report, budget = null) {
       try {
-        const held = await withRepositoryLock(workspace, repoId, () => {
-          for (const work of works) {
+        const held = await withRepositoryLock(workspace, repoId, async () => {
+          for (const [index, work] of works.entries()) {
+            if (budget !== null && index > 0) await budget.checkpoint('settle-operation', true)
             try { report.outcomes.push(settleOne(workspace, work.item, work)) } catch (error) {
               if (error?.crashSeam === true || !isTyped(error)) throw error
               report.outcomes.push({ status: 'failed', code: error.code, editId: work.item.editId, repoId, nodeId: work.item.nodeId, adapterOperationId: work.item.adapterOperationId, state: null, proposalId: null })
@@ -422,6 +455,26 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
       }
     }
 
+    // One observation, from opening the workspace to the flag it leaves for the next.
+    async function observeNow(context, { retryRefused = false } = {}) {
+      const workspace = open(context)
+      const edits = Array.isArray(context.edits) ? context.edits : []
+      // A private note of a personal workspace lives in a generation the personal-workspace module owns: its edit is
+      // not observed, recorded or proposed, and stays held in the vault. Answered by name, with nothing written.
+      const overlayRepoId = personalOverlayRepoOf(workspace.project)
+      const isPrivate = (edit) => overlayRepoId !== null && edit?.identity?.repoId === overlayRepoId
+      const privateNotes = edits.filter((edit) => isPrivate(edit) && edit.closedAt === null)
+        .map((edit) => ({ editId: edit.editId, repoId: edit.identity.repoId, nodeId: edit.identity.nodeId, status: 'refused', code: 'personal-overlay-not-proposed' }))
+      const shared = edits.filter((edit) => !isPrivate(edit))
+      // The graph is built only when an edit is looked at, as for any project. When the personal workspace's graph is
+      // pending, the edits it reached are not reported or recorded, and are offered again on the next observation.
+      const retry = retryRefused || observePendingAgain
+      const looked = await observation.run.cooperatively(workspace, { edits: shared, retryRefused: retry })
+      const pending = looked.filter((item) => item.code === PERSONAL_VALIDATION_PENDING)
+      observePendingAgain = pending.length > 0
+      return { adapterId: PROPOSAL_ADAPTER_ID, observed: [...privateNotes, ...looked.filter((item) => item.code !== PERSONAL_VALIDATION_PENDING)], ...(pending.length > 0 ? { deferred: PERSONAL_VALIDATION_PENDING } : {}) }
+    }
+
     return {
       id: PROPOSAL_ADAPTER_ID,
 
@@ -429,10 +482,10 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
       // observed from its preserved bytes against the source as it is now and recorded as what it is, a bounded number
       // per tick and never twice. `retryRefused` offers once more the edits whose observation refused before anything
       // could be recorded. Writes no source and no vault. Returns codes and identifiers only.
-      async observe(context, { retryRefused = false } = {}) {
-        const workspace = open(context)
-        const edits = Array.isArray(context.edits) ? context.edits : []
-        return { adapterId: PROPOSAL_ADAPTER_ID, observed: observation.run(workspace, { edits, retryRefused }) }
+      observe(context, options = {}) {
+        const run = observing.then(() => observeNow(context, options))
+        observing = run.catch(() => {})
+        return run
       },
 
       // The tick. `context` is { project, workspaceRoot, workspaceId, repositoryRoots, edits }: the pending edit
@@ -442,6 +495,9 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
         const nowMs = Date.parse(isoTime(clock))
         const report = { adapterId: PROPOSAL_ADAPTER_ID, examined: 0, outcomes: [], repositories: [] }
         const edits = Array.isArray(context.edits) ? context.edits : []
+        // The tick goes on answering while it routes: a turn of the event loop when describing has taken its share
+        // of time, and one before each repository and between two of its operations.
+        const budget = createCooperativeBudget()
         const { operations: heads, unreadable } = workspace.queue().heads()
         for (const item of unreadable) report.repositories.push({ repoId: null, code: item.code })
         const byEdit = new Map(edits.map((edit) => [edit.editId, edit]))
@@ -458,7 +514,8 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
           const editState = edit === null || edit.closedAt === null ? 'open' : ['withdrawn', 'superseded'].includes(edit.state) ? edit.state : 'closed'
           let item = itemOfHead(head, edit)
           if (edit !== null && head.state !== 'submitted') {
-            try { item = { ...item, ...(describe(workspace, edit) ?? {}) } } catch (error) { if (!isTyped(error)) throw error; report.outcomes.push({ status: 'failed', code: error.code, editId: head.editId, repoId: head.repoId, nodeId: head.nodeId, adapterOperationId: head.adapterOperationId, state: head.state, proposalId: null }); continue }
+            try { item = { ...item, ...(describe(workspace, edit) ?? {}) } } catch (error) { if (!isTyped(error)) throw error; report.outcomes.push({ status: 'failed', code: error.code, editId: head.editId, repoId: head.repoId, nodeId: head.nodeId, adapterOperationId: head.adapterOperationId, state: head.state, proposalId: null }); await budget.checkpoint('describe-operation'); continue }
+            await budget.checkpoint('describe-operation')
           }
           add(head.repoId, { item, editState })
         }
@@ -471,11 +528,16 @@ export function createProposalAdapterForOracleTests(primitives = PROPOSAL_ADAPTE
         for (const edit of looked) {
           report.examined += 1
           let item
-          try { item = describe(workspace, edit) } catch (error) { if (!isTyped(error)) throw error; report.outcomes.push({ status: 'failed', code: error.code, editId: edit.editId, repoId: edit.identity.repoId, nodeId: edit.identity.nodeId, adapterOperationId: null, state: null, proposalId: null }); continue }
+          try { item = describe(workspace, edit) } catch (error) { if (!isTyped(error)) throw error; report.outcomes.push({ status: 'failed', code: error.code, editId: edit.editId, repoId: edit.identity.repoId, nodeId: edit.identity.nodeId, adapterOperationId: null, state: null, proposalId: null }); await budget.checkpoint('describe-edit'); continue }
           if (item !== null) add(item.repoId, { item, editState: 'open' })
+          await budget.checkpoint('describe-edit')
         }
 
-        for (const [repoId, list] of [...works].sort(([left], [right]) => compareText(left, right))) await settleRepository(workspace, repoId, list, report)
+        for (const [repoId, list] of [...works].sort(([left], [right]) => compareText(left, right))) {
+          await budget.checkpoint('settle-repository', true)
+          await settleRepository(workspace, repoId, list, report, budget)
+        }
+        budget.finish('propose-finalize')
         return report
       },
 

package/src/projection/obsidian/proposals/observation.mjs

@@ -10,6 +10,7 @@ import { SOURCE_APPLY_PRIMITIVES, SourceApplyRefusal, locateSource } from '../ed
 import { EditArbitrationRefusal } from '../edits/arbitrate.mjs'
 import { createEditObserverForOracleTests, editIdempotencyKey, observeEdit } from '../edits/observe.mjs'
 import { decideApply } from '../edits/policy.mjs'
+import { createCooperativeBudget } from '../materialize/prepare-view.mjs'
 import { PublicationRefusal, readFileBytes, sha256Digest } from '../recovery/store.mjs'
 import { PROPOSAL_BACKPRESSURE } from './backpressure.mjs'
 
@@ -38,6 +39,14 @@ import { PROPOSAL_BACKPRESSURE } from './backpressure.mjs'
 // reconciliation cadence, never on every tick. A bounded number of edits is
 // looked at per tick, in the order they were observed, starting where the last
 // tick stopped.
+//
+// One edit is one unit: it is read, judged and recorded without interruption.
+// Between two edits the cooperative form (`cooperatively`, which the adapter
+// uses on a tick) gives the event loop a turn, so the process that runs the
+// tick goes on answering while a burst of edits is observed. A second
+// cooperative observation waits for the first to finish; what is looked at,
+// and in what order, is the same in both forms. The uninterrupted form
+// cannot wait, so it refuses to run beside a cooperative one.
 
 const EXT = OBSIDIAN_EXT_KEY
 const GONE = new Set(['ENOENT', 'ENOTDIR', 'ELOOP'])
@@ -126,6 +135,11 @@ export function createTickObservation(options = {}) {
   // The note exactly as it was published, which the lens needs to tell generated bytes from authored ones. The
   // publisher keeps no copy of a note it created, so the note is prepared again from the sources as they are now and
   // used only when it has the digest the manifest recorded; a retained object of that digest serves as well.
+  //
+  // A source replaced between the look at its path and its open (ELEAFCHANGED) is another program writing it, as it
+  // is to the publisher and to source apply: the preparation answers null, and the caller reads the source again and
+  // refuses stale-source when it changed. Only a preparation that succeeded is kept for its scope and generation: one
+  // failure tells of one moment and is never the answer for the other edits of that generation in the tick.
   function publishedNoteOf(workspace, { scope, manifest, noteEntry }, now) {
     const store = workspace.recoveryOf(scope.scopeId)
     const kept = retained(store, noteEntry.noteDigest)
@@ -134,7 +148,7 @@ export function createTickObservation(options = {}) {
     // Prepared in the vault layout of the edit's generation, once per generation.
     const preparedKey = `${scope.scopeId}\u0000${manifest.generationId}`
     if (!workspace.prepared.has(preparedKey)) {
-      let files = []
+      let files
       try {
         const { graph, profile } = workspace.corpus()
         const snapshot = seams.captureSnapshot({ project: workspace.project, graph, workspaceId: workspace.workspaceId, index: new Map(), configDigest: manifest.ext?.[EXT]?.configDigest ?? `sha256:${'0'.repeat(64)}`, capturedAt: now })
@@ -143,7 +157,10 @@ export function createTickObservation(options = {}) {
           snapshot, profile, scope, persistentPathRegistry, priorManifest: manifest, existingSettings: null, clock: workspace.clock,
           vaultRootBytes: Buffer.byteLength(store.vaultRoot, 'utf8'), layout: manifestLayoutVersion(manifest),
         }).files
-      } catch (error) { if (!isTyped(error) && !GONE.has(error?.code)) throw error }
+      } catch (error) {
+        if (!isTyped(error) && error?.code !== 'ELEAFCHANGED' && !GONE.has(error?.code)) throw error
+        return null
+      }
       workspace.prepared.set(preparedKey, files)
     }
     const file = workspace.prepared.get(preparedKey).find((item) => item.path === noteEntry.path && item.digest === noteEntry.noteDigest)
@@ -195,7 +212,8 @@ export function createTickObservation(options = {}) {
   // `workspace` is the opened workspace of the adapter; `edits` the pending edit records, open and closed;
   // `retryRefused` offers the edits whose observation refused earlier once more. Returns one entry per edit looked
   // at: { editId, repoId, nodeId, status: 'observed' | 'refused', code, kind?, operationState?, idempotencyKey? }.
-  return function observePendingEdits(workspace, { edits, retryRefused = false }) {
+  // Yields between two edits, never within one, and returns the entries when it is done.
+  function* observeSteps(workspace, { edits, retryRefused = false }) {
     if (retryRefused) unobservable.clear()
     const now = isoTime(workspace.clock)
     const candidates = edits.filter((edit) => edit.closedAt === null && OPEN_EDIT_STATES.includes(edit.state) && !recorded.has(edit.editId) && !unobservable.has(edit.editId))
@@ -204,7 +222,8 @@ export function createTickObservation(options = {}) {
     const looked = [...candidates.slice(start), ...candidates.slice(0, start)].slice(0, bounds.maxObservedPerTick)
     cursor = candidates.length > bounds.maxObservedPerTick ? start + looked.length : 0
     const observed = []
-    for (const edit of looked) {
+    for (const [index, edit] of looked.entries()) {
+      if (index > 0) yield 'observe-edit'
       const entry = { editId: edit.editId, repoId: edit.identity.repoId, nodeId: edit.identity.nodeId }
       try {
         const outcome = observeOne(workspace, edit, now)
@@ -218,4 +237,31 @@ export function createTickObservation(options = {}) {
     }
     return observed
   }
+
+  // Cooperative observations asked for and not yet finished, and the last of them.
+  let underWay = 0
+  let last = Promise.resolve()
+  function observePendingEdits(workspace, input) {
+    if (underWay > 0) throw Object.assign(new Error('a cooperative observation of this instance is under way; the uninterrupted form cannot run beside it'), { code: 'observation-under-way' })
+    const steps = observeSteps(workspace, input)
+    for (;;) { const step = steps.next(); if (step.done) return step.value }
+  }
+  // The same observation with a turn of the event loop between two edits. One at a time: an observation asked for
+  // while another is under way starts when that one has finished, so no edit is looked at by both. One that failed
+  // holds back none that follow.
+  observePendingEdits.cooperatively = (workspace, input) => {
+    underWay += 1
+    const run = last.then(async () => {
+      const budget = createCooperativeBudget()
+      const steps = observeSteps(workspace, input)
+      for (;;) {
+        const step = steps.next()
+        if (step.done) { budget.finish('observe-finalize'); return step.value }
+        await budget.checkpoint(step.value, true)
+      }
+    }).finally(() => { underWay -= 1 })
+    last = run.catch(() => {})
+    return run
+  }
+  return observePendingEdits
 }