amalgm 0.1.220 → 0.1.234

package/runtime/scripts/amalgm-mcp/workspace/wall.js

@@ -24,13 +24,14 @@ const {
   readCheckpoint,
   repoKeyOfCard,
   sameTransition,
+  stampCheckpointMutation,
 } = require('./checkpoint');
+const projectStream = require('./project-stream');
 
-const WALL_CONTRACT = 'code-project@4';
+const { WALL_CONTRACT } = projectStream;
 const PROJECT_DIR = path.join(AMALGM_DIR, 'state', 'code-projects');
 const REGISTRY_FILE = path.join(PROJECT_DIR, 'projects.json');
 const PUBLISH_QUIET_MS = 25;
-const COMPACT_AT_BYTES = 1024 * 1024;
 const ENSURE_STEP_TIMEOUT_MS = 20_000;
 const ENSURE_INTERVAL_MS = 15_000;
 const ATTACH_REJECTED = /shared_resource_read_forbidden/;
@@ -46,10 +47,6 @@ const service = {
   lastPublishedAt: null,
 };
 
-function sha256(text) {
-  return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
-}
-
 function readRegistry() {
   try {
     const parsed = JSON.parse(fs.readFileSync(REGISTRY_FILE, 'utf8'));
@@ -67,14 +64,14 @@ function writeRegistry(registry) {
 function projectResourceId(repoKey, ownerUserId = userId()) {
   if (!repoKey || !ownerUserId) return null;
   const digest = crypto.createHash('sha256')
-    .update(`amalgm-code-project@1|${ownerUserId}|${repoKey}`)
+    .update(`amalgm-code-project@2|${ownerUserId}|${repoKey}`)
     .digest('base64url');
   return `shr_cp1${digest.slice(0, 40)}`;
 }
 
 function projectFile(resourceId) {
   const digest = crypto.createHash('sha256').update(resourceId).digest('hex').slice(0, 24);
-  return path.join(PROJECT_DIR, `${digest}.jsonl`);
+  return path.join(PROJECT_DIR, `${digest}.json`);
 }
 
 function projectForRepo(repoKey) {
@@ -132,7 +129,8 @@ function projects() {
 }
 
 function wallReplica(project) {
-  return require('../state/replicas').getReplicaByResourceId(project.resourceId);
+  const replica = require('../state/replicas').getReplicaByResourceId(project.resourceId);
+  return replica?.contract === projectStream.STREAM_CONTRACT ? replica : null;
 }
 
 function publicationStatus(repoKey, reduced = undefined) {
@@ -158,57 +156,38 @@ function publicationStatus(repoKey, reduced = undefined) {
   };
 }
 
-function parseWallLine(line) {
-  try {
-    const entry = JSON.parse(line);
-    if (
-      entry?.wall !== WALL_CONTRACT
-      || !entry.project
-      || !entry.machine
-      || !entry.mutationId
-      || !Array.isArray(entry.cards)
-    ) return null;
-    return entry;
-  } catch {
-    return null;
-  }
-}
-
-function sealCardVersions(project, card) {
+function sealCardVersions(project, card, record = null) {
+  const versionFor = (mutationId) => (
+    mutationId === record?.mutationId && validVersion(record?.version)
+      ? record.version
+      : officialVersion(project.resourceId, mutationId)
+  );
   return {
     ...card,
     positionVersion: validVersion(card.positionVersion)
       ? card.positionVersion
-      : officialVersion(project.resourceId, card.positionMutationId),
+      : versionFor(card.positionMutationId),
     branches: (card.branches || []).map((ref) => ({
       ...ref,
       version: validVersion(ref.version)
         ? ref.version
-        : officialVersion(project.resourceId, ref.mutationId),
+        : versionFor(ref.mutationId),
     })),
     checkpoints: (card.checkpoints || []).map((checkpoint) => ({
       ...checkpoint,
       version: validVersion(checkpoint.version)
         ? checkpoint.version
-        : officialVersion(project.resourceId, checkpoint.mutationId),
+        : versionFor(checkpoint.mutationId),
     })),
   };
 }
 
 function readProjectWall(project) {
-  if (!fs.existsSync(project.file)) return [];
-  const entries = new Map();
-  for (const line of require('../state/docs').getDocText(project.file).split('\n')) {
-    if (!line) continue;
-    const entry = parseWallLine(line);
-    if (entry?.project !== project.repoKey) continue;
-    entries.set(entry.machine, {
-      ...entry,
-      resourceId: project.resourceId,
-      cards: entry.cards.map((card) => sealCardVersions(project, card)),
-    });
-  }
-  return [...entries.values()];
+  return projectStream.records(project).map((entry) => ({
+    ...entry,
+    resourceId: project.resourceId,
+    cards: entry.cards.map((card) => sealCardVersions(project, card, entry)),
+  }));
 }
 
 function readWall(repoKey = null) {
@@ -228,7 +207,6 @@ function publishedCardFor(card) {
     || published.head?.branch !== card.head?.branch
     || published.head?.oid !== card.head?.oid
     || Boolean(published.head?.detached) !== Boolean(card.head?.detached)
-    || hasObjectUpgrade(card, { card: published, branches: published.branches })
   ) return null;
   return published;
 }
@@ -304,16 +282,17 @@ function hasLocalContribution(card, plan, baseline, checkpoint = null, previousC
   return false;
 }
 
-function hasObjectUpgrade(card, plan) {
-  if (!plan?.card || !samePosition(card, plan.card)) return false;
-  if (card.objects?.kind === 'bundle' && plan.card.objects?.kind !== 'bundle') return true;
+function hasPendingContributionObjects(card, plan) {
+  if (!plan?.card || !samePosition(card, plan.card)) {
+    return card.objects?.kind === 'pending';
+  }
   const shared = new Map((plan.branches || []).map((ref) => [ref.name, ref]));
   return (card.refs || []).some((ref) => {
-    const onRail = shared.get(ref.name);
-    return onRail
-      && onRail.oid === ref.oid
-      && ref.objects?.kind === 'bundle'
-      && onRail.objects?.kind !== 'bundle';
+    const accepted = shared.get(ref.name);
+    const alreadyAvailable = accepted
+      && validVersion(accepted.version)
+      && (accepted.oid ?? null) === (ref.oid ?? null);
+    return !alreadyAvailable && ref.objects?.kind === 'pending';
   });
 }
 
@@ -348,16 +327,6 @@ function checkpointBaseFor(card, baseline, project, shared = null) {
   };
 }
 
-function hasCheckpointBaseUpgrade(card, plan, baseline, project) {
-  const shared = plan?.card?.checkpointBase;
-  const candidate = checkpointBaseFor(card, baseline, project, shared);
-  return Boolean(shared && candidate) && (
-    candidate.through === shared.through
-    && candidate.value?.revision === shared.value?.revision
-    && JSON.stringify(candidate.value) !== JSON.stringify(shared.value)
-  );
-}
-
 function validVersion(value) {
   return Number.isInteger(value) && value > 0;
 }
@@ -373,6 +342,30 @@ function officialVersion(resourceId, mutationId) {
   return Number.isInteger(mutation?.officialCloudVersion) ? mutation.officialCloudVersion : null;
 }
 
+function durableMutationVersion(project, mutationId) {
+  if (!project || !mutationId) return null;
+  for (const record of projectStream.records(project)) {
+    if (record.mutationId === mutationId && validVersion(record.version)) return record.version;
+    for (const card of record.cards) {
+      if (card.positionMutationId === mutationId) {
+        return validVersion(card.positionVersion) ? card.positionVersion : null;
+      }
+      const nested = [
+        ...(card.branches || []),
+        ...(card.checkpoints || []),
+      ].find((entry) => entry.mutationId === mutationId);
+      if (nested) return validVersion(nested.version) ? nested.version : null;
+    }
+  }
+  return null;
+}
+
+function mutationVersion(resourceId, mutationId, repoKey = null) {
+  if (!resourceId) return null;
+  return officialVersion(resourceId, mutationId)
+    || durableMutationVersion(repoKey ? projectForRepo(repoKey) : null, mutationId);
+}
+
 function sealedVersion(resourceId, mutationId, ...versions) {
   return versions.find(validVersion)
     || officialVersion(resourceId, mutationId)
@@ -537,41 +530,18 @@ function stampCards(cards, previousCards, sharedPlan, baseline, mutationId, proj
   });
 }
 
-function withoutPublishedAt(line) {
-  const entry = parseWallLine(line);
-  if (!entry) return line;
-  const { publishedAt, mutationId, ...rest } = entry;
-  return JSON.stringify(rest);
+function cardsUseMutation(cards, mutationId) {
+  return cards.some((card) => (
+    card.positionMutationId === mutationId
+    || (card.branches || []).some((ref) => ref.mutationId === mutationId)
+    || (card.checkpoints || []).some((checkpoint) => checkpoint.mutationId === mutationId)
+  ));
 }
 
-/**
- * Keep every machine's newest complete record and delete the largest span
- * containing none of them. Repeating this as the wall grows removes all
- * obsolete history, even when a retired machine's last record is near the
- * beginning. Each pass is one deletion, so concurrent appends commute in Yjs.
- */
-function compactWallText(current, repoKey) {
-  const latest = new Map();
-  const lines = current.split('\n');
-  let offset = 0;
-  for (let index = 0; index < lines.length; index += 1) {
-    const line = lines[index];
-    const entry = line ? parseWallLine(line) : null;
-    const end = offset + line.length + (index < lines.length - 1 ? 1 : 0);
-    if (entry?.project === repoKey) latest.set(entry.machine, { start: offset, end });
-    offset = end;
-  }
-  if (latest.size === 0) return current;
-  const records = [...latest.values()].sort((left, right) => left.start - right.start);
-  const gaps = [{ start: 0, end: records[0].start }];
-  for (let index = 0; index < records.length - 1; index += 1) {
-    gaps.push({ start: records[index].end, end: records[index + 1].start });
-  }
-  const gap = gaps.reduce((largest, candidate) => (
-    candidate.end - candidate.start > largest.end - largest.start ? candidate : largest
-  ));
-  if (gap.end <= gap.start) return current;
-  return `${current.slice(0, gap.start)}${current.slice(gap.end)}`;
+function samePublishedRecord(left, right) {
+  if (!left || !right) return false;
+  const stripTransport = ({ publishedAt, mutationId, version, ...record }) => record;
+  return JSON.stringify(stripTransport(left)) === JSON.stringify(stripTransport(right));
 }
 
 function publishProject(project, rawCards) {
@@ -584,33 +554,26 @@ function publishProject(project, rawCards) {
     service.counters.publishSkips += 1;
     return { published: false, reason: 'repo-materializing', repoKey: project.repoKey };
   }
-  const docs = require('../state/docs');
-  let current = docs.getDocText(project.file);
-  let previous = null;
-  let previousEntry = null;
-  for (const line of current.split('\n')) {
-    if (!line) continue;
-    const entry = parseWallLine(line);
-    if (entry?.project === project.repoKey && entry.machine === machineId()) {
-      previous = line;
-      previousEntry = entry;
-    }
-  }
+  const previousEntry = projectStream.records(project)
+    .find((entry) => entry.machine === machineId()) || null;
 
   const reducer = require('./reducer');
   const sharedPlan = reducer.reduceWall(readProjectWall(project)).get(project.repoKey);
   const baseline = reducer.readRepoState(project.repoKey);
-  const previousCards = (previousEntry?.cards || []).map((card) => sealCardVersions(project, card));
+  const previousCards = (previousEntry?.cards || [])
+    .map((card) => sealCardVersions(project, card, previousEntry));
   const previousReplicas = new Set(previousCards.map((card) => card.replica));
   const currentReplicas = new Set(rawCards.map((card) => card.replica));
   const removed = [...previousReplicas].some((replica) => !currentReplicas.has(replica));
   const contributes = rawCards.some((card) => {
     const checkpoint = readCheckpoint(machineId(), project.repoKey);
     const previousCard = previousCards.find((entry) => entry.replica === card.replica);
-    return hasLocalContribution(card, sharedPlan, baseline, checkpoint, previousCard)
-      || hasObjectUpgrade(card, sharedPlan)
-      || hasCheckpointBaseUpgrade(card, sharedPlan, baseline, project);
+    return hasLocalContribution(card, sharedPlan, baseline, checkpoint, previousCard);
   });
+  if (contributes && rawCards.some((card) => hasPendingContributionObjects(card, sharedPlan))) {
+    service.counters.publishSkips += 1;
+    return { published: false, reason: 'objects-preparing', repoKey: project.repoKey };
+  }
   if (!removed && !contributes) {
     service.counters.publishSkips += 1;
     return { published: false, reason: 'shared-state-covered', repoKey: project.repoKey };
@@ -624,20 +587,22 @@ function publishProject(project, rawCards) {
     mutationId,
     project,
   );
-  const mine = JSON.stringify({
+  const mine = {
     wall: WALL_CONTRACT,
     project: project.repoKey,
     machine: machineId(),
     mutationId,
     publishedAt: new Date().toISOString(),
     cards,
-  });
-  const unchanged = previous !== null
-    && withoutPublishedAt(previous) === withoutPublishedAt(mine);
-  if (unchanged) {
+  };
+  if (samePublishedRecord(previousEntry, mine)) {
     service.counters.publishSkips += 1;
     return { published: false, reason: 'unchanged', repoKey: project.repoKey };
   }
+  if (!cardsUseMutation(cards, mutationId)) {
+    service.counters.publishSkips += 1;
+    return { published: false, reason: 'no-content-change', repoKey: project.repoKey };
+  }
   const status = publicationStatus(project.repoKey, baseline);
   if (!status.current) {
     service.counters.publishSkips += 1;
@@ -649,31 +614,22 @@ function publishProject(project, rawCards) {
     };
   }
 
-  const separator = current && !current.endsWith('\n') ? '\n' : '';
-  if (Buffer.byteLength(`${current}${separator}${mine}\n`, 'utf8') > COMPACT_AT_BYTES) {
-    const compacted = compactWallText(current, project.repoKey);
-    if (compacted !== current) {
-      docs.writeDocText(project.file, compacted, {
-        baseHash: sha256(current),
-        mutationId: `mut_project_compact_${crypto.randomBytes(16).toString('base64url')}`,
-      });
-      current = docs.getDocText(project.file);
+  for (const card of cards) {
+    const checkpoint = (card.checkpoints || []).find((entry) => (
+      entry.machine === machineId() && entry.mutationId === mutationId
+    ));
+    if (checkpoint) {
+      stampCheckpointMutation(machineId(), project.repoKey, checkpoint, mutationId);
     }
   }
-  const nextSeparator = current && !current.endsWith('\n') ? '\n' : '';
-  const result = docs.writeDocText(
-    project.file,
-    `${current}${nextSeparator}${mine}\n`,
-    { baseHash: sha256(current), mutationId },
-  );
+  const result = projectStream.publish(project, mine);
   service.counters.publishes += 1;
   service.lastPublishedAt = new Date().toISOString();
   return {
     published: true,
     repoKey: project.repoKey,
     resourceId: project.resourceId,
-    mutationId: result.mutationId,
-    merge: result.merge,
+    mutationId: result.mutation.mutationId,
     cards: cards.length,
   };
 }
@@ -748,6 +704,7 @@ async function ensureProject(project) {
     await callLocal(shared.handleAttachmentStart, {
       resourceId: project.resourceId,
       path: project.file,
+      contract: projectStream.STREAM_CONTRACT,
     });
   } else if (request.state === 'rejected') {
     if (
@@ -758,6 +715,9 @@ async function ensureProject(project) {
       await callLocal(shared.handlePromotionStart, {
         resourceId: project.resourceId,
         path: project.file,
+        project: project.repoKey,
+        contract: projectStream.STREAM_CONTRACT,
+        schemaVersion: 1,
       });
       await waitForSettled(project, () => promotions.getPromotionByResource(project.resourceId));
     }
@@ -802,13 +762,17 @@ function startWallService() {
   if (service.started) return;
   service.started = true;
   service.unsubscribe = subscribeStateEvents((event) => {
-    const acknowledged = event.resource === 'shared-mutations'
-      && projects().some((project) => project.resourceId === event.sharedResourceId);
+    const acknowledgedProject = event.resource === 'shared-mutations'
+      ? projects().find((project) => project.resourceId === event.sharedResourceId)
+      : null;
+    if (acknowledgedProject) {
+      projectStream.seal(acknowledgedProject, event.mutationId || event.id, event.resourceVersion);
+    }
     if (
       event.resource === 'repo-cards'
       || event.resource === 'repo-checkpoints'
       || event.resource === 'repo-objects'
-      || acknowledged
+      || acknowledgedProject
     ) {
       ensureProjects().catch(() => {});
       schedulePublish();
@@ -865,12 +829,17 @@ function wallStats() {
 }
 
 module.exports = {
-  _test: { publishSelected },
+  _test: {
+    cardsUseMutation,
+    hasPendingContributionObjects,
+    publishSelected,
+    samePublishedRecord,
+  },
   WALL_CONTRACT,
   bindProject,
-  compactWallText,
   hasLocalContribution,
   isCommittedCard,
+  mutationVersion,
   projectFile,
   projectForRepo,
   projectResourceId,