amalgm 0.1.238 → 0.1.240

package/runtime/scripts/amalgm-mcp/state/doc-disk.js

@@ -0,0 +1,550 @@
+'use strict';
+
+/**
+ * Physical boundary for live-document files.
+ *
+ * The Yjs mutation journal proves semantic order. This table and the native
+ * file receipt prove which exact disk predecessor a materialization replaced.
+ * Machine-local disk evidence never enters the cloud operation.
+ */
+
+const crypto = require('crypto');
+const fs = require('fs');
+const path = require('path');
+const { cachedStatement, ensureDocStatesSchema, openLocalDb } = require('./db');
+const nativeFileTransaction = require('../workspace/native-file-transaction/client');
+
+const CONTRACT = 'doc-disk-materialization@1';
+
+function byteIdentity(content, stat = null) {
+  if (content === null) return null;
+  const bytes = Buffer.isBuffer(content) ? content : Buffer.from(content, 'utf8');
+  return {
+    kind: 'file',
+    exec: stat ? (stat.mode & 0o100n) !== 0n : false,
+    bytes: bytes.length,
+    sha256: crypto.createHash('sha256').update(bytes).digest('hex'),
+    ...(stat ? {
+      dev: stat.dev.toString(),
+      ino: stat.ino.toString(),
+    } : {}),
+  };
+}
+
+function observe(filePath, testHooks = {}) {
+  for (let attempt = 0; attempt < 8; attempt += 1) {
+    const parent = fs.realpathSync(path.dirname(filePath));
+    const rootStat = fs.statSync(parent, { bigint: true });
+    const relativePath = path.basename(filePath);
+    const result = nativeFileTransaction.observe({
+      root: parent,
+      rootDev: rootStat.dev.toString(),
+      rootIno: rootStat.ino.toString(),
+      path: relativePath,
+    }, testHooks);
+    if (result.status === 'changed') continue;
+    if (result.status !== 'observed') {
+      throw new Error(`unexpected native document observation: ${result.status}`);
+    }
+    const content = result.content === undefined
+      ? null
+      : Buffer.from(result.content, 'base64');
+    const identity = result.value
+      ? {
+        ...result.value,
+        exec: result.value.exec === true,
+        bytes: Number(result.value.bytes || 0),
+      }
+      : null;
+    return {
+      root: parent,
+      rootDev: result.rootDev,
+      rootIno: result.rootIno,
+      relativePath: result.path,
+      text: content === null ? null : content.toString('utf8'),
+      identity,
+    };
+  }
+  throw new Error(`live document kept changing during observation: ${filePath}`);
+}
+
+function sameIdentity(left, right) {
+  return JSON.stringify(left) === JSON.stringify(right);
+}
+
+function sameObservation(left, right) {
+  return Boolean(
+    left
+    && right
+    && left.rootDev === right.rootDev
+    && left.rootIno === right.rootIno
+    && left.relativePath === right.relativePath
+    && sameIdentity(left.identity, right.identity),
+  );
+}
+
+function transactionId(base, channelId, mutationId) {
+  return crypto.createHash('sha256')
+    .update(
+      `${CONTRACT}\0${base.rootDev}\0${base.rootIno}\0${base.relativePath}`
+      + `\0${channelId}\0${mutationId}`,
+    )
+    .digest('hex');
+}
+
+function validateBaseObservation(base) {
+  if (base.text === null) {
+    if (base.identity !== null) {
+      throw new Error('missing document base has a physical identity');
+    }
+    return;
+  }
+  if (typeof base.text !== 'string' || base.identity?.kind !== 'file') {
+    throw new Error('document disk base is incomplete');
+  }
+  const contentIdentity = byteIdentity(Buffer.from(base.text, 'utf8'));
+  if (
+    contentIdentity.bytes !== base.identity.bytes
+    || contentIdentity.sha256 !== base.identity.sha256
+  ) {
+    throw new Error('document disk base text does not match its physical identity');
+  }
+}
+
+function insertBase(database, mutation, base, mutationSource = 'doc:recovery') {
+  ensureDocStatesSchema(database);
+  validateBaseObservation(base);
+  const id = transactionId(base, mutation.channelId, mutation.mutationId);
+  cachedStatement(database, `
+    INSERT INTO doc_disk_materializations (
+      channel_id,
+      mutation_id,
+      local_resource,
+      root,
+      root_dev,
+      root_ino,
+      relative_path,
+      transaction_id,
+      operation_id,
+      expected_json,
+      observed_text,
+      mutation_source
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(channel_id, mutation_id) DO NOTHING
+  `).run(
+    mutation.channelId,
+    mutation.mutationId,
+    mutation.localResource,
+    base.root,
+    base.rootDev,
+    base.rootIno,
+    base.relativePath,
+    id,
+    `${mutation.channelId}:${mutation.mutationId}`,
+    base.identity === null ? null : JSON.stringify(base.identity),
+    base.text,
+    mutationSource,
+  );
+}
+
+function normalize(row) {
+  if (!row) return null;
+  return {
+    channelId: row.channel_id,
+    mutationId: row.mutation_id,
+    localResource: row.local_resource,
+    root: row.root,
+    rootDev: row.root_dev,
+    rootIno: row.root_ino,
+    relativePath: row.relative_path,
+    transactionId: row.transaction_id,
+    operationId: row.operation_id,
+    expected: row.expected_json === null ? null : JSON.parse(row.expected_json),
+    baseText: row.observed_text,
+    mutationSource: row.mutation_source || (
+      row.observed_text === null ? 'doc:recovery' : 'doc:disk'
+    ),
+    target: row.target_json === null ? null : JSON.parse(row.target_json),
+    targetContent: row.target_content === null ? null : Buffer.from(row.target_content),
+    planKind: row.plan_kind,
+    snapshotPayload: row.snapshot_payload_json === null
+      ? null
+      : JSON.parse(row.snapshot_payload_json),
+    snapshotEventSeq: row.snapshot_event_seq === null
+      ? null
+      : Number(row.snapshot_event_seq),
+    state: row.materialization_state,
+    successorChannelId: row.successor_channel_id,
+    successorMutationId: row.successor_mutation_id,
+    createdAt: row.created_at,
+  };
+}
+
+function snapshotMutation(payload, base) {
+  const digest = crypto.createHash('sha256')
+    .update(
+      `${CONTRACT}\0snapshot\0${payload.resourceId}\0${payload.snapshotVersion}`
+      + `\0${payload.snapshotChecksum}\0${base.rootDev}\0${base.rootIno}`
+      + `\0${base.relativePath}\0${JSON.stringify(base.identity)}`,
+    )
+    .digest('hex');
+  return {
+    channelId: `snapshot:${payload.resourceId}`,
+    mutationId: `snapshot_${digest}`,
+    localResource: payload.localResource,
+  };
+}
+
+function insertSnapshotBase(database, payload, base) {
+  const mutation = snapshotMutation(payload, base);
+  insertBase(database, mutation, base, 'doc:cloud-snapshot');
+  cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET plan_kind = 'snapshot',
+        snapshot_payload_json = ?,
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE channel_id = ?
+      AND mutation_id = ?
+      AND plan_kind = 'mutation'
+      AND materialization_state = 'based'
+  `).run(JSON.stringify(payload), mutation.channelId, mutation.mutationId);
+  const plan = get(mutation.channelId, mutation.mutationId, database);
+  if (!plan || plan.planKind !== 'snapshot') {
+    throw new Error(`snapshot ${payload.resourceId}@${payload.snapshotVersion} changed its disk plan`);
+  }
+  return plan;
+}
+
+function firstBase(localResource, database = openLocalDb()) {
+  ensureDocStatesSchema(database);
+  return normalize(cachedStatement(database, `
+    SELECT * FROM doc_disk_materializations
+    WHERE local_resource = ?
+    ORDER BY created_at ASC, mutation_id ASC
+    LIMIT 1
+  `).get(localResource) || null);
+}
+
+function get(channelId, mutationId, database = openLocalDb()) {
+  ensureDocStatesSchema(database);
+  return normalize(cachedStatement(database, `
+    SELECT * FROM doc_disk_materializations
+    WHERE channel_id = ? AND mutation_id = ?
+  `).get(channelId, mutationId) || null);
+}
+
+function listSnapshots(localResource, database = openLocalDb()) {
+  ensureDocStatesSchema(database);
+  return cachedStatement(database, `
+    SELECT * FROM doc_disk_materializations
+    WHERE local_resource = ? AND plan_kind = 'snapshot'
+    ORDER BY created_at ASC, mutation_id ASC
+  `).all(localResource).map(normalize);
+}
+
+function listMutations(localResource, database = openLocalDb()) {
+  ensureDocStatesSchema(database);
+  return cachedStatement(database, `
+    SELECT * FROM doc_disk_materializations
+    WHERE local_resource = ? AND plan_kind = 'mutation'
+    ORDER BY created_at ASC, mutation_id ASC
+  `).all(localResource).map(normalize);
+}
+
+function prepare(channelId, mutationId, targetText, database = openLocalDb()) {
+  const held = get(channelId, mutationId, database);
+  if (!held) throw new Error(`document mutation ${mutationId} has no disk base`);
+  const content = Buffer.from(targetText, 'utf8');
+  const target = byteIdentity(content);
+  target.exec = held.expected?.exec === true;
+  if (held.target) {
+    if (!sameIdentity(held.target, target) || !held.targetContent.equals(content)) {
+      throw new Error(`document mutation ${mutationId} changed its disk target`);
+    }
+    return held;
+  }
+  cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET target_json = ?,
+        target_content = ?,
+        materialization_state = 'prepared',
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE channel_id = ? AND mutation_id = ? AND materialization_state = 'based'
+  `).run(JSON.stringify(target), content, channelId, mutationId);
+  return get(channelId, mutationId, database);
+}
+
+function transaction(plan) {
+  return {
+    root: plan.root,
+    rootDev: plan.rootDev,
+    rootIno: plan.rootIno,
+    path: plan.relativePath,
+    id: plan.transactionId,
+    operation: plan.operationId,
+    expected: plan.expected,
+    target: plan.target,
+  };
+}
+
+function scratchPath(plan) {
+  return path.join(plan.root, `.amalgm-apply-${plan.transactionId}`);
+}
+
+function cross(plan, database = openLocalDb(), testHooks = {}) {
+  if (!plan?.target || !plan.targetContent) {
+    throw new Error(`document mutation ${plan?.mutationId || 'unknown'} has no prepared target`);
+  }
+  let result;
+  if (fs.existsSync(scratchPath(plan))) {
+    result = nativeFileTransaction.run(transaction(plan), 'recover');
+  } else {
+    result = nativeFileTransaction.run(
+      transaction(plan),
+      'apply',
+      plan.targetContent,
+      testHooks,
+    );
+  }
+  if (result.status === 'applied') {
+    cachedStatement(database, `
+      UPDATE doc_disk_materializations
+      SET materialization_state = 'crossed',
+          updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+      WHERE channel_id = ? AND mutation_id = ?
+    `).run(plan.channelId, plan.mutationId);
+  }
+  return result;
+}
+
+function markSuperseded(
+  database,
+  channelId,
+  mutationId,
+  successorChannelId,
+  successorMutationId,
+) {
+  const result = cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET materialization_state = 'superseded',
+        successor_channel_id = ?,
+        successor_mutation_id = ?,
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE channel_id = ?
+      AND mutation_id = ?
+      AND materialization_state IN ('based', 'prepared')
+  `).run(successorChannelId, successorMutationId, channelId, mutationId);
+  if (result.changes !== 1) {
+    throw new Error(`document mutation ${mutationId} could not retain its disk successor`);
+  }
+}
+
+function linkCrossedSuccessor(
+  database,
+  channelId,
+  mutationId,
+  successorChannelId,
+  successorMutationId,
+) {
+  const result = cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET successor_channel_id = ?,
+        successor_mutation_id = ?,
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE channel_id = ?
+      AND mutation_id = ?
+      AND materialization_state = 'crossed'
+      AND (
+        successor_channel_id IS NULL
+        OR (
+          successor_channel_id = ?
+          AND successor_mutation_id = ?
+        )
+      )
+  `).run(
+    successorChannelId,
+    successorMutationId,
+    channelId,
+    mutationId,
+    successorChannelId,
+    successorMutationId,
+  );
+  if (result.changes !== 1) {
+    throw new Error(`crossed document mutation ${mutationId} changed its disk successor`);
+  }
+}
+
+function rebaseExpected(channelId, mutationId, observation, database = openLocalDb()) {
+  validateBaseObservation(observation);
+  const held = get(channelId, mutationId, database);
+  if (!held || held.state !== 'prepared') {
+    throw new Error(`document mutation ${mutationId} cannot refresh its disk predecessor`);
+  }
+  if (fs.existsSync(scratchPath(held))) {
+    throw new Error(`document mutation ${mutationId} still has a physical receipt`);
+  }
+  const id = transactionId(observation, channelId, mutationId);
+  cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET root = ?,
+        root_dev = ?,
+        root_ino = ?,
+        relative_path = ?,
+        transaction_id = ?,
+        expected_json = ?,
+        observed_text = ?,
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE channel_id = ? AND mutation_id = ? AND materialization_state = 'prepared'
+  `).run(
+    observation.root,
+    observation.rootDev,
+    observation.rootIno,
+    observation.relativePath,
+    id,
+    observation.identity === null ? null : JSON.stringify(observation.identity),
+    observation.text,
+    channelId,
+    mutationId,
+  );
+  return get(channelId, mutationId, database);
+}
+
+function rebindLocalResource(
+  database,
+  {
+    oldLocalResource,
+    newLocalResource,
+    newPath,
+  },
+) {
+  const root = fs.realpathSync(path.dirname(newPath));
+  const rootStat = fs.statSync(root, { bigint: true });
+  const rootDev = rootStat.dev.toString();
+  const rootIno = rootStat.ino.toString();
+  const relativePath = path.basename(newPath);
+  cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET local_resource = ?,
+        root = CASE
+          WHEN root_dev = ? AND root_ino = ? THEN ?
+          ELSE root
+        END,
+        relative_path = CASE
+          WHEN root_dev = ? AND root_ino = ? THEN ?
+          ELSE relative_path
+        END,
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE local_resource = ? OR local_resource = ?
+  `).run(
+    newLocalResource,
+    rootDev,
+    rootIno,
+    root,
+    rootDev,
+    rootIno,
+    relativePath,
+    oldLocalResource,
+    newLocalResource,
+  );
+}
+
+function rehomeMutation(
+  database,
+  {
+    mutationId,
+    oldChannelId,
+    newChannelId,
+  },
+) {
+  if (oldChannelId === newChannelId) return;
+  cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET channel_id = ?,
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE channel_id = ? AND mutation_id = ?
+  `).run(newChannelId, oldChannelId, mutationId);
+  cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET successor_channel_id = ?,
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE successor_channel_id = ? AND successor_mutation_id = ?
+  `).run(newChannelId, oldChannelId, mutationId);
+}
+
+function cancel(channelId, mutationId, database = openLocalDb()) {
+  const plan = get(channelId, mutationId, database);
+  if (!plan) return;
+  if (plan.state !== 'based' && plan.state !== 'prepared') {
+    throw new Error(`document disk plan ${mutationId} cannot cancel from ${plan.state}`);
+  }
+  if (fs.existsSync(scratchPath(plan))) {
+    throw new Error(`document disk plan ${mutationId} still has a physical receipt`);
+  }
+  cachedStatement(database, `
+    DELETE FROM doc_disk_materializations
+    WHERE channel_id = ? AND mutation_id = ?
+  `).run(channelId, mutationId);
+}
+
+function markSnapshotCommitted(channelId, mutationId, eventSeq, database = openLocalDb()) {
+  const result = cachedStatement(database, `
+    UPDATE doc_disk_materializations
+    SET snapshot_event_seq = ?,
+        updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+    WHERE channel_id = ?
+      AND mutation_id = ?
+      AND plan_kind = 'snapshot'
+      AND materialization_state = 'crossed'
+      AND snapshot_event_seq IS NULL
+  `).run(eventSeq, channelId, mutationId);
+  if (result.changes !== 1) {
+    const held = get(channelId, mutationId, database);
+    if (!held || held.snapshotEventSeq !== Number(eventSeq)) {
+      throw new Error(`snapshot disk plan ${mutationId} did not commit its event`);
+    }
+  }
+}
+
+function finish(channelId, mutationId, database = openLocalDb()) {
+  const plan = get(channelId, mutationId, database);
+  if (!plan) return;
+  if (plan.state === 'crossed') {
+    if (plan.planKind === 'snapshot' && plan.snapshotEventSeq === null) {
+      throw new Error(`snapshot disk plan ${mutationId} has not committed its cursor`);
+    }
+    const result = nativeFileTransaction.run(transaction(plan), 'release');
+    if (result.status !== 'released') {
+      throw new Error(
+        `document disk receipt ${mutationId} did not release: ${result.reason || result.status}`,
+      );
+    }
+  } else if (plan.state !== 'superseded') {
+    throw new Error(`document disk plan ${mutationId} cannot finish from ${plan.state}`);
+  }
+  cachedStatement(database, `
+    DELETE FROM doc_disk_materializations
+    WHERE channel_id = ? AND mutation_id = ?
+  `).run(channelId, mutationId);
+}
+
+module.exports = {
+  byteIdentity,
+  cancel,
+  cross,
+  finish,
+  firstBase,
+  get,
+  insertBase,
+  insertSnapshotBase,
+  linkCrossedSuccessor,
+  listMutations,
+  listSnapshots,
+  markSuperseded,
+  markSnapshotCommitted,
+  observe,
+  prepare,
+  rebindLocalResource,
+  rebaseExpected,
+  rehomeMutation,
+  sameObservation,
+};