@1aboveio/skills 0.10.0

package/skills/engineering/resolve-release/scripts/durable-processing.mjs

@@ -0,0 +1,522 @@
+#!/usr/bin/env node
+
+import crypto from 'node:crypto'
+import fs from 'node:fs'
+import path from 'node:path'
+import { CheckedAdapterError, loadCheckedAdapter } from './checked-adapter-loader.mjs'
+import { isMainModule } from '../../engineering-runtime/scripts/main-module.mjs'
+
+export const ADAPTER_EXPORT = 'DURABLE_PROCESSING_ADAPTER'
+export const ADAPTER_SCHEMA_VERSION = 1
+export const RECEIPT_SCHEMA = 'resolve-release/durable-processing-receipt@1'
+export const PAUSE_ORDER = Object.freeze(['database-admission', 'worker-scheduling'])
+export const RESUME_ORDER = Object.freeze([...PAUSE_ORDER].reverse())
+
+const CLI_WRITER = Object.freeze({ id: 'resolve-release/durable-processing' })
+
+const USAGE = `durable-processing.mjs - drive a repository-declared durable-processing adapter
+
+Usage:
+  durable-processing.mjs preflight --declaration <release.json>
+  durable-processing.mjs status --declaration <release.json> [options]
+  durable-processing.mjs pause --declaration <release.json> [options]
+  durable-processing.mjs quiescence --declaration <release.json> [options]
+  durable-processing.mjs resume --declaration <release.json> [options]
+  durable-processing.mjs down --declaration <release.json> [options]
+  durable-processing.mjs up --declaration <release.json> [options]
+  durable-processing.mjs unwind-pre-shift --declaration <release.json> [options]
+
+The declaration must decide durableProcessing.present as true or false. A true
+declaration names an ES module exporting DURABLE_PROCESSING_ADAPTER. The adapter
+contains typed data and an execute(request) function; declarations contain no shell
+commands, credentials, timestamps, or platform-specific snippets. The adapter is a
+dependency-free strict-UTF-8 .mjs file no larger than 256 KiB. Execution receives
+only the declared config and isolated snapshot-backed readTextFile/writeTextFile operations;
+validated final write intents commit in the host only after receipt validation. The host stages
+same-directory final and original files, validates every target identity immediately before the
+first deterministic replacement, and restores replaced targets in reverse if a later operation
+fails. Rollback failure is reported as ambiguous with affected targets. This recoverable
+transaction cannot provide cross-file kernel atomicity or guaranteed recovery from process crash
+or power loss during the replacement window.`
+
+const isObject = (value) => value != null && typeof value === 'object' && !Array.isArray(value)
+const nonEmptyString = (value) => typeof value === 'string' && value.trim() !== ''
+const safeGeneration = (value) => Number.isSafeInteger(value) && value >= 0
+const isoTimestamp = (value) => nonEmptyString(value) && Number.isFinite(Date.parse(value))
+const exactArray = (actual, expected) => Array.isArray(actual)
+  && actual.length === expected.length
+  && actual.every((value, index) => value === expected[index])
+
+function written(state, fields = {}) {
+  return {
+    schema: RECEIPT_SCHEMA,
+    state,
+    ...fields,
+    writer: CLI_WRITER,
+    writtenAt: new Date().toISOString(),
+  }
+}
+
+function refused(...failed) {
+  return written('refused', { failed: [...new Set(failed)] })
+}
+
+function readJson(file, failedId) {
+  try {
+    const value = JSON.parse(fs.readFileSync(file, 'utf8'))
+    return { value }
+  } catch (error) {
+    return { error: refused(failedId), cause: error }
+  }
+}
+
+function validateAdapter(adapter) {
+  const failed = []
+  if (!isObject(adapter) || adapter.schemaVersion !== ADAPTER_SCHEMA_VERSION) {
+    failed.push('e:invalid-adapter-schema-version')
+  }
+  if (!nonEmptyString(adapter?.writerId)) failed.push('e:invalid-writer-id')
+  if (!exactArray(adapter?.pauseOrder, PAUSE_ORDER)) failed.push('e:invalid-pause-order')
+  if (!exactArray(adapter?.resumeOrder, RESUME_ORDER)) failed.push('e:resume-not-exact-reverse')
+  const quiescence = adapter?.quiescence
+  if (
+    !isObject(quiescence)
+    || !nonEmptyString(quiescence.component)
+    || !nonEmptyString(quiescence.blockerField)
+    || typeof quiescence.readyValue !== 'number'
+    || !Number.isSafeInteger(quiescence.timeoutMs)
+    || quiescence.timeoutMs <= 0
+  ) failed.push('e:invalid-quiescence')
+  if (adapter?.evidenceSchema !== RECEIPT_SCHEMA) failed.push('e:invalid-evidence-schema')
+  if (typeof adapter?.execute !== 'function') failed.push('e:adapter-not-executable')
+  return failed
+}
+
+export async function loadDeclaration(file, { onFilesystemBoundary } = {}) {
+  const read = readJson(file, 'e:declaration-unreadable')
+  if (read.error) return read
+  const declaration = read.value
+  if (!isObject(declaration) || declaration.schemaVersion !== 1) {
+    return { error: refused('e:invalid-declaration-schema') }
+  }
+  if (!nonEmptyString(declaration.repositoryRoot)) {
+    return { error: refused('e:repository-root-undeclared') }
+  }
+  const durable = declaration.durableProcessing
+  if (!isObject(durable) || typeof durable.present !== 'boolean') {
+    return { error: refused('e:durable-processing-undecided') }
+  }
+  if (durable.config != null && !isObject(durable.config)) {
+    return { error: refused('e:adapter-config-invalid') }
+  }
+  if (durable.present === false) return { declaration, durable }
+  if (!nonEmptyString(durable.adapter)) return { error: refused('e:adapter-undeclared') }
+
+  const declarationDir = path.dirname(path.resolve(file))
+  const repositoryRoot = path.isAbsolute(declaration.repositoryRoot)
+    ? path.resolve(declaration.repositoryRoot)
+    : path.resolve(declarationDir, declaration.repositoryRoot)
+  let loaded
+  try {
+    loaded = await loadCheckedAdapter({
+      repositoryRoot,
+      adapterPath: durable.adapter,
+      onFilesystemBoundary,
+    })
+  } catch (error) {
+    const code = error instanceof CheckedAdapterError ? error.code : 'e:adapter-unloadable'
+    return { error: refused(code) }
+  }
+  const adapter = loaded.module[ADAPTER_EXPORT]
+  const failed = validateAdapter(adapter)
+  if (failed.length) return { error: refused(...failed) }
+  return { declaration, durable, adapter, adapterPath: loaded.canonicalPath }
+}
+
+export async function preflight(file) {
+  const loaded = await loadDeclaration(file)
+  if (loaded.error) return loaded.error
+  if (loaded.durable.present === false) return written('ready', { present: false })
+  const { adapter, adapterPath } = loaded
+  return written('ready', {
+    present: true,
+    adapter: adapterPath,
+    adapterSchemaVersion: adapter.schemaVersion,
+    pauseOrder: [...adapter.pauseOrder],
+    resumeOrder: [...adapter.resumeOrder],
+    quiescence: { ...adapter.quiescence },
+    evidenceSchema: adapter.evidenceSchema,
+    adapterWriter: { id: adapter.writerId },
+  })
+}
+
+export function normalizePlatformOutput(output) {
+  if (!isObject(output)) return { error: 'e:platform-output-malformed' }
+  const hasPayload = Object.hasOwn(output, 'payload')
+  const hasText = Object.hasOwn(output, 'text')
+  if (hasPayload === hasText) return { error: 'e:platform-output-ambiguous' }
+  if (hasPayload) {
+    if (!isObject(output.payload)) return { error: 'e:platform-output-malformed' }
+    return { receipt: output.payload }
+  }
+  if (!nonEmptyString(output.text)) return { error: 'e:platform-output-malformed' }
+  try {
+    const receipt = JSON.parse(output.text)
+    return isObject(receipt) ? { receipt } : { error: 'e:platform-output-malformed' }
+  } catch {
+    return { error: 'e:platform-output-malformed' }
+  }
+}
+
+export function validateReceipt(receipt, request, writerId) {
+  const required = [
+    'schema', 'state', 'operation', 'component', 'requestId', 'writer', 'writtenAt',
+    'generation', 'fence', 'observed',
+  ]
+  if (!isObject(receipt) || required.some((field) => receipt[field] == null)) {
+    return ['e:receipt-partial']
+  }
+  const failed = []
+  if (receipt.schema !== RECEIPT_SCHEMA) failed.push('e:receipt-schema-mismatch')
+  if (!['observed', 'applied', 'refused'].includes(receipt.state)) failed.push('e:receipt-state-invalid')
+  if (receipt.operation !== request.operation) failed.push('e:receipt-operation-mismatch')
+  if (receipt.component !== request.component) failed.push('e:receipt-component-mismatch')
+  if (receipt.requestId !== request.requestId) failed.push('e:receipt-request-mismatch')
+  if (!isObject(receipt.writer) || receipt.writer.id !== writerId) failed.push('e:receipt-writer-mismatch')
+  if (!isoTimestamp(receipt.writtenAt)) failed.push('e:receipt-timestamp-invalid')
+  if (!safeGeneration(receipt.generation)) failed.push('e:receipt-generation-invalid')
+  if (!nonEmptyString(receipt.fence)) failed.push('e:receipt-fence-invalid')
+  if (!isObject(receipt.observed)) failed.push('e:receipt-observation-invalid')
+  if (receipt.state === 'refused' && (
+    !Array.isArray(receipt.failed) || receipt.failed.length === 0 || receipt.failed.some((id) => !nonEmptyString(id))
+  )) failed.push('e:receipt-partial')
+  if (receipt.state !== 'refused') {
+    if (['pause', 'resume'].includes(request.operation) && receipt.state !== 'applied') {
+      failed.push('e:receipt-effect-unproven')
+    }
+    if (request.operation === 'pause' && receipt.observed?.paused !== true) failed.push('e:receipt-effect-unproven')
+    if (request.operation === 'resume' && receipt.observed?.resumed !== true) failed.push('e:receipt-effect-unproven')
+    if (['status', 'quiescence'].includes(request.operation) && receipt.state !== 'observed') {
+      failed.push('e:receipt-observation-unproven')
+    }
+    if (request.operation === 'quiescence'
+      && (!Number.isSafeInteger(receipt.observed?.blockers) || receipt.observed.blockers < 0)) {
+      failed.push('e:receipt-observation-incomplete')
+    }
+    if (request.operation === 'status' && (
+      typeof receipt.observed?.admissionPaused !== 'boolean'
+      || typeof receipt.observed?.schedulingPaused !== 'boolean'
+      || !Number.isSafeInteger(receipt.observed?.blockers)
+      || receipt.observed.blockers < 0
+      || !isObject(receipt.observed?.traffic)
+    )) failed.push('e:receipt-observation-incomplete')
+  }
+  return failed
+}
+
+export async function executeOperation(file, operation, options) {
+  const loaded = await loadDeclaration(file)
+  if (loaded.error) return loaded.error
+  if (loaded.durable.present === false) return refused('e:durable-processing-not-present')
+  const request = Object.freeze({
+    schemaVersion: 1,
+    operation,
+    component: options.component,
+    requestId: options.requestId ?? crypto.randomUUID(),
+    expectedGeneration: options.generation,
+    expectedFence: options.fence,
+    servingRevision: options.servingRevision,
+  })
+  if (!nonEmptyString(request.component) || !nonEmptyString(request.requestId)) {
+    return refused('e:request-invalid')
+  }
+  let isolated
+  try {
+    isolated = await loaded.adapter.execute(
+      request,
+      Object.freeze({ ...(loaded.durable.config ?? {}) }),
+    )
+  } catch (error) {
+    const code = typeof error?.code === 'string' && error.code.startsWith('e:adapter-')
+      ? error.code
+      : 'e:adapter-execution-failed'
+    return refused(code)
+  }
+  if (!isObject(isolated) || !Object.hasOwn(isolated, 'output')
+      || typeof isolated.commit !== 'function') {
+    return refused('e:adapter-execution-failed')
+  }
+  const normalized = normalizePlatformOutput(isolated.output)
+  if (normalized.error) return refused(normalized.error)
+  const failed = validateReceipt(normalized.receipt, request, loaded.adapter.writerId)
+  if (failed.length) return refused(...failed)
+  try {
+    isolated.commit()
+  } catch (error) {
+    const code = typeof error?.code === 'string' && error.code.startsWith('e:adapter-')
+      ? error.code
+      : 'e:adapter-file-changed'
+    if (error?.ambiguousEffects === true || error?.committedEffects === true) {
+      return written('refused', {
+        failed: [code],
+        effects: error.ambiguousEffects === true ? 'ambiguous' : 'committed',
+        affectedTargets: Array.isArray(error.affectedTargets) ? [...error.affectedTargets] : [],
+      })
+    }
+    return refused(code)
+  }
+  return normalized.receipt
+}
+
+function sequenceReceipt(state, fields) {
+  return written(state, fields)
+}
+
+function isSuccess(receipt) {
+  return receipt.state === 'observed' || receipt.state === 'applied'
+}
+
+function identityFrom(receipt) {
+  return { generation: receipt.generation, fence: receipt.fence }
+}
+
+async function status(file) {
+  return executeOperation(file, 'status', { component: 'durable-state' })
+}
+
+export async function down(file, options) {
+  const release = readJson(options.releaseState, 'e:release-state-unreadable')
+  if (release.error) return release.error
+  if (release.value?.skill !== 'resolve-release' || release.value?.step !== 'expose') {
+    return refused('e:expose-step-not-recorded')
+  }
+  const initial = await status(file)
+  if (!isSuccess(initial)) return initial
+  const receipts = []
+  let identity = identityFrom(initial)
+  for (const component of PAUSE_ORDER) {
+    const receipt = await executeOperation(file, 'pause', { component, ...identity })
+    receipts.push(receipt)
+    if (!isSuccess(receipt)) return sequenceReceipt('blocked', {
+      failed: receipt.failed,
+      pausedAt: receipts.find((item) => isSuccess(item))?.writtenAt,
+      pauseOrder: [...PAUSE_ORDER],
+      receipts,
+      proof: receipt,
+    })
+    identity = identityFrom(receipt)
+  }
+  const loaded = await loadDeclaration(file)
+  if (loaded.error) return loaded.error
+  const deadline = Date.now() + loaded.adapter.quiescence.timeoutMs
+  let proof
+  do {
+    proof = await executeOperation(file, 'quiescence', {
+      component: 'durable-state',
+      ...identity,
+    })
+    receipts.push(proof)
+    if (!isSuccess(proof)) return sequenceReceipt('blocked', {
+      failed: proof.failed,
+      pausedAt: receipts[0]?.writtenAt,
+      pauseOrder: [...PAUSE_ORDER],
+      receipts,
+    })
+    if (proof.observed.blockers === 0) break
+    if (Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 25))
+  } while (Date.now() < deadline)
+  if (proof.observed.blockers !== 0) {
+    return sequenceReceipt('blocked', {
+      failed: ['e:quiescence-timeout'],
+      pausedAt: receipts[0]?.writtenAt,
+      pauseOrder: [...PAUSE_ORDER],
+      receipts,
+      proof,
+    })
+  }
+  return sequenceReceipt('ready-to-shift', {
+    pausedAt: receipts[0]?.writtenAt,
+    pauseOrder: [...PAUSE_ORDER],
+    receipts,
+    proof,
+    shiftFence: identityFrom(proof),
+  })
+}
+
+function trafficMatches(traffic, expected) {
+  return isObject(traffic)
+    && traffic.incumbentRevision === expected.incumbentRevision
+    && traffic.incumbentPercent === expected.incumbentPercent
+    && traffic.candidateRevision === expected.candidateRevision
+    && traffic.candidatePercent === expected.candidatePercent
+}
+
+async function resumeSequence(file, options, initialStatus) {
+  const receipts = [initialStatus]
+  let identity = identityFrom(initialStatus)
+  if (
+    !safeGeneration(options.generation)
+    || !nonEmptyString(options.fence)
+    || options.generation !== identity.generation
+    || options.fence !== identity.fence
+  ) {
+    return sequenceReceipt('ambiguous', {
+      failed: ['e:serving-generation-mismatch'],
+      receipts,
+    })
+  }
+  for (const component of RESUME_ORDER) {
+    const receipt = await executeOperation(file, 'resume', {
+      component,
+      ...identity,
+      servingRevision: options.servingRevision,
+    })
+    receipts.push(receipt)
+    if (!isSuccess(receipt)) {
+      return sequenceReceipt('ambiguous', { failed: receipt.failed, receipts })
+    }
+    identity = identityFrom(receipt)
+  }
+  const confirmation = await status(file)
+  receipts.push(confirmation)
+  if (
+    !isSuccess(confirmation)
+    || confirmation.observed.schedulingPaused !== false
+    || confirmation.observed.admissionPaused !== false
+  ) {
+    return sequenceReceipt('ambiguous', {
+      failed: ['e:processing-restoration-ambiguous'],
+      receipts,
+    })
+  }
+  return sequenceReceipt('restored', {
+    resumedAt: receipts.find((receipt) => receipt.operation === 'resume')?.writtenAt,
+    resumeOrder: [...RESUME_ORDER],
+    receipts,
+    servingRevisionAtResume: options.servingRevision,
+    restorationFence: identityFrom(confirmation),
+  })
+}
+
+export async function up(file, options) {
+  const observed = await status(file)
+  if (!isSuccess(observed)) return observed
+  const traffic = observed.observed.traffic
+  const identitiesDeclared = nonEmptyString(options.incumbentRevision)
+    && nonEmptyString(options.candidateRevision)
+    && options.incumbentRevision !== options.candidateRevision
+  const namedCandidateServes = options.servingRevision === options.candidateRevision
+    && trafficMatches(traffic, {
+      incumbentRevision: options.incumbentRevision,
+      incumbentPercent: 0,
+      candidateRevision: options.candidateRevision,
+      candidatePercent: 100,
+    })
+  const namedIncumbentServes = options.servingRevision === options.incumbentRevision
+    && trafficMatches(traffic, {
+      incumbentRevision: options.incumbentRevision,
+      incumbentPercent: 100,
+      candidateRevision: options.candidateRevision,
+      candidatePercent: 0,
+    })
+  if (!identitiesDeclared || (!namedCandidateServes && !namedIncumbentServes)) {
+    return sequenceReceipt('ambiguous', {
+      failed: ['e:serving-revision-unproven'],
+      trafficProof: traffic,
+      receipts: [observed],
+    })
+  }
+  const resumed = await resumeSequence(file, options, observed)
+  return resumed.state === 'restored'
+    ? {
+        ...resumed,
+        trafficProof: traffic,
+        incumbentRevision: options.incumbentRevision,
+        candidateRevision: options.candidateRevision,
+      }
+    : resumed
+}
+
+export async function unwindPreShift(file, options) {
+  const observed = await status(file)
+  if (!isSuccess(observed)) return observed
+  const trafficProof = observed.observed.traffic
+  const expected = {
+    incumbentRevision: options.incumbentRevision,
+    incumbentPercent: 100,
+    candidateRevision: options.candidateRevision,
+    candidatePercent: 0,
+  }
+  if (!trafficMatches(trafficProof, expected)) {
+    return sequenceReceipt('ambiguous', {
+      failed: ['e:traffic-split-ambiguous'],
+      trafficProof,
+      receipts: [observed],
+    })
+  }
+  const resumed = await resumeSequence(file, {
+    ...options,
+    servingRevision: options.incumbentRevision,
+  }, observed)
+  if (resumed.state !== 'restored') return resumed
+  return { ...resumed, state: 'unwound', trafficProof }
+}
+
+function parseArgs(argv) {
+  const positional = []
+  const options = {}
+  const unknown = []
+  for (let index = 0; index < argv.length; index++) {
+    const value = argv[index]
+    if (value === '--help' || value === '-h') options.help = true
+    else if (value === '--declaration') options.declaration = argv[++index]
+    else if (value === '--component') options.component = argv[++index]
+    else if (value === '--generation') options.generation = Number(argv[++index])
+    else if (value === '--fence') options.fence = argv[++index]
+    else if (value === '--serving-revision') options.servingRevision = argv[++index]
+    else if (value === '--request-id') options.requestId = argv[++index]
+    else if (value === '--release-state') options.releaseState = argv[++index]
+    else if (value === '--incumbent-revision') options.incumbentRevision = argv[++index]
+    else if (value === '--candidate-revision') options.candidateRevision = argv[++index]
+    else if (value.startsWith('--')) unknown.push(value)
+    else positional.push(value)
+  }
+  return { positional, options, unknown }
+}
+
+export async function main(argv) {
+  const { positional, options, unknown } = parseArgs(argv)
+  if (options.help || positional[0] === 'help') {
+    console.log(USAGE)
+    return 0
+  }
+  if (unknown.length) {
+    console.log(JSON.stringify(refused('e:unknown-option')))
+    return 2
+  }
+  if (!nonEmptyString(options.declaration)) {
+    console.log(JSON.stringify(refused('e:declaration-required')))
+    return 2
+  }
+  const command = positional[0]
+  const operations = ['status', 'pause', 'quiescence', 'resume']
+  const sequences = ['down', 'up', 'unwind-pre-shift']
+  if (command !== 'preflight' && !operations.includes(command) && !sequences.includes(command)) {
+    console.log(JSON.stringify(refused('e:unknown-command')))
+    return 2
+  }
+  let receipt
+  if (command === 'preflight') receipt = await preflight(options.declaration)
+  else if (operations.includes(command)) receipt = await executeOperation(options.declaration, command, options)
+  else if (command === 'down') receipt = await down(options.declaration, options)
+  else if (command === 'up') receipt = await up(options.declaration, options)
+  else receipt = await unwindPreShift(options.declaration, options)
+  console.log(JSON.stringify(receipt))
+  return ['ready', 'observed', 'applied', 'ready-to-shift', 'restored', 'unwound'].includes(receipt.state)
+    ? 0
+    : 1
+}
+
+if (isMainModule(import.meta.url)) {
+  main(process.argv.slice(2)).then((code) => { process.exitCode = code })
+}