amalgm 0.1.238 → 0.1.240

package/runtime/scripts/amalgm-mcp/workspace/objects.js

@@ -14,12 +14,17 @@ const fs = require('fs');
 const path = require('path');
 const { AMALGM_DIR } = require('../config');
 const { appendStateEvent } = require('../state/events');
-const { originContains } = require('./git');
+const durable = require('./durable');
 const transition = require('./transition');
+const {
+  CHUNK_BYTES,
+  describeBytes,
+  manifestChecksum,
+  sha256,
+} = require('./content');
 
-const CONTRACT = 'git-bundle-chunks@1';
-const PAYLOAD_CONTRACT = 'opaque-chunks@1';
-const CHUNK_BYTES = 512 * 1024;
+const CONTRACT = 'git-bundle-chunks@2';
+const PAYLOAD_CONTRACT = 'opaque-chunks@2';
 const ROOT = path.join(AMALGM_DIR, 'state', 'repo-objects');
 const MANIFESTS = path.join(ROOT, 'manifests');
 const PAYLOADS = path.join(ROOT, 'payloads');
@@ -27,8 +32,16 @@ const CHUNKS = path.join(ROOT, 'chunks');
 const REQUESTS = path.join(ROOT, 'requests');
 const preparing = new Map();
 
-function sha256(bytes) {
-  return crypto.createHash('sha256').update(bytes).digest('hex');
+function isProjectId(value) {
+  return require('./tree/identity').isUuid(String(value || ''));
+}
+
+function locatorChecksum(chunks) {
+  return manifestChecksum(chunks);
+}
+
+function locatorBytes(chunks) {
+  return chunks.reduce((total, chunk) => total + chunk.bytes, 0);
 }
 
 function ensureDirs() {
@@ -56,8 +69,7 @@ function readJson(file) {
 }
 
 function writeJson(file, value) {
-  fs.mkdirSync(path.dirname(file), { recursive: true });
-  fs.writeFileSync(file, `${JSON.stringify(value)}\n`, { mode: 0o600 });
+  durable.writeJson(file, value);
 }
 
 function publicLocator(manifest) {
@@ -83,14 +95,18 @@ function publicPayloadLocator(manifest) {
   };
 }
 
-function writeChunks(bytes) {
-  const chunks = [];
-  for (let offset = 0; offset < bytes.length; offset += CHUNK_BYTES) {
-    const part = bytes.subarray(offset, Math.min(offset + CHUNK_BYTES, bytes.length));
-    const digest = sha256(part);
-    const chunkFile = path.join(CHUNKS, `${digest}.bin`);
-    if (!fs.existsSync(chunkFile)) fs.writeFileSync(chunkFile, part, { mode: 0o600 });
-    chunks.push({ sha256: digest, bytes: part.length, uploaded: false });
+function describeChunks(bytes) {
+  return describeBytes(bytes).map((chunk) => ({ ...chunk, uploaded: false }));
+}
+
+function writeChunks(bytes, chunks = describeChunks(bytes)) {
+  let offset = 0;
+  for (const chunk of chunks) {
+    const part = bytes.subarray(offset, offset + chunk.bytes);
+    const chunkFile = path.join(CHUNKS, `${chunk.sha256}.bin`);
+    const held = fs.existsSync(chunkFile) ? fs.readFileSync(chunkFile) : null;
+    if (!held || sha256(held) !== chunk.sha256) durable.writeFile(chunkFile, part);
+    offset += chunk.bytes;
   }
   return chunks;
 }
@@ -113,8 +129,8 @@ async function prepareBundle(worktreeRoot, repoKey, resourceId, oid, file) {
       repoKey,
       oid,
       ref,
-      checksum: sha256(bundle),
-      bytes: bundle.length,
+      checksum: locatorChecksum(chunks),
+      bytes: locatorBytes(chunks),
       chunks,
       createdAt: new Date().toISOString(),
     });
@@ -128,23 +144,29 @@ function publishPayload(key, bytesInput, repoKey) {
   ensureDirs();
   const bytes = Buffer.isBuffer(bytesInput) ? bytesInput : Buffer.from(bytesInput);
   const project = require('./wall').projectForRepo(repoKey);
-  const checksum = sha256(bytes);
   if (!project?.resourceId) {
-    return { kind: 'pending', contract: PAYLOAD_CONTRACT, checksum };
+    return { kind: 'pending', contract: PAYLOAD_CONTRACT };
   }
+  const chunks = describeChunks(bytes);
+  const checksum = locatorChecksum(chunks);
   // Payload content is its identity. A later checkpoint that still contains
   // the same large file must reference the already-uploaded bytes instead of
   // creating another upload merely because the surrounding checkpoint moved.
   const file = payloadFile(project.resourceId, checksum);
   let manifest = readJson(file);
-  if (!manifest || manifest.checksum !== checksum) {
+  if (
+    !manifest
+    || manifest.contract !== PAYLOAD_CONTRACT
+    || manifest.checksum !== checksum
+  ) {
+    writeChunks(bytes, chunks);
     manifest = {
       contract: PAYLOAD_CONTRACT,
       resourceId: project.resourceId,
       repoKey,
       checksum,
-      bytes: bytes.length,
-      chunks: writeChunks(bytes),
+      bytes: locatorBytes(chunks),
+      chunks,
       createdAt: new Date().toISOString(),
     };
     writeJson(file, manifest);
@@ -182,7 +204,6 @@ async function bundleSourceFor(worktreeRoot, repoKey, oid) {
 
 async function objectSourceFor(worktreeRoot, repoKey, oid) {
   if (!oid || !repoKey) return null;
-  if (await originContains(worktreeRoot, oid)) return { kind: 'remote' };
   return bundleSourceFor(worktreeRoot, repoKey, oid);
 }
 
@@ -190,10 +211,22 @@ function claimUpload() {
   ensureDirs();
   for (const directory of [MANIFESTS, PAYLOADS]) {
     for (const name of fs.readdirSync(directory).sort()) {
-      const manifest = readJson(path.join(directory, name));
+      const file = path.join(directory, name);
+      const manifest = readJson(file);
       const chunk = manifest?.chunks?.find((entry) => !entry.uploaded);
       if (!chunk) continue;
-      const bytes = fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`));
+      const chunkFile = path.join(CHUNKS, `${chunk.sha256}.bin`);
+      const bytes = fs.existsSync(chunkFile) ? fs.readFileSync(chunkFile) : null;
+      if (
+        !bytes
+        || bytes.length !== chunk.bytes
+        || sha256(bytes) !== chunk.sha256
+      ) {
+        fs.rmSync(chunkFile, { force: true });
+        fs.rmSync(file, { force: true });
+        wake('delete', chunk.sha256);
+        continue;
+      }
       return {
         resourceId: manifest.resourceId,
         sha256: chunk.sha256,
@@ -243,17 +276,20 @@ function validatePayloadLocator(locator) {
   if (
     locator?.kind !== 'chunks'
     || locator?.contract !== PAYLOAD_CONTRACT
-    || !/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(locator.resourceId || ''))
+    || !isProjectId(locator.resourceId)
     || !/^[0-9a-f]{64}$/.test(String(locator.checksum || ''))
     || !Number.isSafeInteger(locator.bytes)
-    || locator.bytes < 0
+    || locator.bytes < 1
     || !Array.isArray(locator.chunks)
+    || locator.chunks.length < 1
     || locator.chunks.some((chunk) => (
       !/^[0-9a-f]{64}$/.test(String(chunk?.sha256 || ''))
       || !Number.isSafeInteger(chunk?.bytes)
-      || chunk.bytes < 0
+      || chunk.bytes < 1
       || chunk.bytes > CHUNK_BYTES
     ))
+    || locatorBytes(locator.chunks) !== locator.bytes
+    || locatorChecksum(locator.chunks) !== locator.checksum
   ) throw new Error('invalid checkpoint payload locator');
 }
 
@@ -263,7 +299,11 @@ function readPayload(locator) {
   let waiting = false;
   for (const chunk of locator.chunks) {
     const file = path.join(CHUNKS, `${chunk.sha256}.bin`);
-    if (fs.existsSync(file) && fs.statSync(file).size === chunk.bytes) continue;
+    if (fs.existsSync(file)) {
+      const held = fs.readFileSync(file);
+      if (held.length === chunk.bytes && sha256(held) === chunk.sha256) continue;
+      fs.rmSync(file, { force: true });
+    }
     writeJson(requestFile(locator.resourceId, chunk.sha256), {
       resourceId: locator.resourceId,
       sha256: chunk.sha256,
@@ -282,7 +322,7 @@ function readPayload(locator) {
     return bytes;
   });
   const bytes = Buffer.concat(parts);
-  if (bytes.length !== locator.bytes || sha256(bytes) !== locator.checksum) {
+  if (bytes.length !== locator.bytes) {
     throw new Error('checkpoint payload verification failed');
   }
   writeJson(payloadFile(locator.resourceId, locator.checksum), {
@@ -303,21 +343,29 @@ function requestFile(resourceId, digest) {
 
 function claimDownload() {
   ensureDirs();
-  const name = fs.readdirSync(REQUESTS).filter((entry) => entry.endsWith('.json')).sort()[0];
-  if (!name) return null;
-  const request = readJson(path.join(REQUESTS, name));
-  return request?.sha256 ? request : null;
+  for (const name of fs.readdirSync(REQUESTS).filter((entry) => entry.endsWith('.json')).sort()) {
+    const file = path.join(REQUESTS, name);
+    const request = readJson(file);
+    if (
+      request
+      && Object.keys(request).length === 2
+      && isProjectId(request.resourceId)
+      && /^[0-9a-f]{64}$/.test(String(request.sha256 || ''))
+    ) return request;
+    fs.rmSync(file, { force: true });
+  }
+  return null;
 }
 
 function receiveDownload(resourceId, digest, dataBase64) {
-  if (!/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(resourceId || ''))) {
+  if (!isProjectId(resourceId)) {
     throw new Error('invalid repo object project');
   }
   if (!/^[0-9a-f]{64}$/.test(String(digest || ''))) throw new Error('invalid repo object checksum');
   ensureDirs();
   const bytes = Buffer.from(String(dataBase64 || ''), 'base64');
   if (sha256(bytes) !== digest) throw new Error('repo object checksum mismatch');
-  fs.writeFileSync(path.join(CHUNKS, `${digest}.bin`), bytes, { mode: 0o600 });
+  durable.writeFile(path.join(CHUNKS, `${digest}.bin`), bytes);
   fs.rmSync(requestFile(resourceId, digest), { force: true });
   wake('update', digest);
   require('./reducer').reduceNow().catch(() => {});
@@ -326,17 +374,24 @@ function receiveDownload(resourceId, digest, dataBase64) {
 
 function validateLocator(locator) {
   if (
-    locator?.contract !== CONTRACT
-    || !/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(locator.resourceId || ''))
-    || !/^[0-9a-f]{40,64}$/.test(String(locator.oid || ''))
+    locator?.kind !== 'bundle'
+    || locator?.contract !== CONTRACT
+    || !isProjectId(locator.resourceId)
+    || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(String(locator.oid || ''))
     || !/^[0-9a-f]{64}$/.test(String(locator.checksum || ''))
+    || locator.ref !== `refs/amalgm/exports/${locator.oid}`
+    || !Number.isSafeInteger(locator.bytes)
+    || locator.bytes < 1
     || !Array.isArray(locator.chunks)
+    || locator.chunks.length < 1
     || locator.chunks.some((chunk) => (
       !/^[0-9a-f]{64}$/.test(String(chunk?.sha256 || ''))
       || !Number.isSafeInteger(chunk?.bytes)
-      || chunk.bytes < 0
+      || chunk.bytes < 1
       || chunk.bytes > CHUNK_BYTES
     ))
+    || locatorBytes(locator.chunks) !== locator.bytes
+    || locatorChecksum(locator.chunks) !== locator.checksum
   ) throw new Error('invalid git bundle locator');
 }
 
@@ -346,7 +401,11 @@ function prepareBundleImport(locator) {
   let waiting = false;
   for (const chunk of locator.chunks) {
     const file = path.join(CHUNKS, `${chunk.sha256}.bin`);
-    if (fs.existsSync(file) && fs.statSync(file).size === chunk.bytes) continue;
+    if (fs.existsSync(file)) {
+      const held = fs.readFileSync(file);
+      if (held.length === chunk.bytes && sha256(held) === chunk.sha256) continue;
+      fs.rmSync(file, { force: true });
+    }
     writeJson(requestFile(locator.resourceId, chunk.sha256), {
       resourceId: locator.resourceId,
       sha256: chunk.sha256,
@@ -359,19 +418,24 @@ function prepareBundleImport(locator) {
   }
 
   const bundleFile = path.join(ROOT, `${locator.checksum}.bundle`);
-  if (!fs.existsSync(bundleFile)) {
-    const parts = locator.chunks.map((chunk) => {
-      const bytes = fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`));
-      if (bytes.length !== chunk.bytes || sha256(bytes) !== chunk.sha256) {
-        throw new Error('git bundle chunk verification failed');
-      }
-      return bytes;
-    });
-    const bundle = Buffer.concat(parts);
-    if (bundle.length !== locator.bytes || sha256(bundle) !== locator.checksum) {
-      throw new Error('git bundle verification failed');
+  const parts = locator.chunks.map((chunk) => (
+    fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`))
+  ));
+  const bundleMatches = fs.existsSync(bundleFile) && (() => {
+    const held = fs.readFileSync(bundleFile);
+    let offset = 0;
+    for (let index = 0; index < locator.chunks.length; index += 1) {
+      const chunk = locator.chunks[index];
+      const part = held.subarray(offset, offset + chunk.bytes);
+      if (part.length !== chunk.bytes || sha256(part) !== chunk.sha256) return false;
+      offset += chunk.bytes;
     }
-    fs.writeFileSync(bundleFile, bundle, { mode: 0o600 });
+    return offset === held.length && held.length === locator.bytes;
+  })();
+  if (!bundleMatches) {
+    const bundle = Buffer.concat(parts);
+    if (bundle.length !== locator.bytes) throw new Error('git bundle verification failed');
+    durable.writeFile(bundleFile, bundle);
   }
   return { state: 'ready', file: bundleFile };
 }
@@ -387,6 +451,7 @@ module.exports = {
   CHUNK_BYTES,
   CONTRACT,
   PAYLOAD_CONTRACT,
+  _test: { describeChunks, locatorBytes, locatorChecksum },
   acknowledgeUpload,
   bundleSourceFor,
   claimDownload,