amalgm 0.1.239 → 0.1.241

package/runtime/scripts/amalgm-mcp/workspace/objects.js

@@ -14,21 +14,36 @@ const fs = require('fs');
 const path = require('path');
 const { AMALGM_DIR } = require('../config');
 const { appendStateEvent } = require('../state/events');
-const { originContains } = require('./git');
+const durable = require('./durable');
 const transition = require('./transition');
+const {
+  CHUNK_BYTES,
+  describeBytes,
+  manifestChecksum,
+  sha256,
+} = require('./content');
 
-const CONTRACT = 'git-bundle-chunks@1';
-const PAYLOAD_CONTRACT = 'opaque-chunks@1';
-const CHUNK_BYTES = 512 * 1024;
+const CONTRACT = 'git-bundle-chunks@2';
+const PAYLOAD_CONTRACT = 'opaque-chunks@2';
 const ROOT = path.join(AMALGM_DIR, 'state', 'repo-objects');
 const MANIFESTS = path.join(ROOT, 'manifests');
 const PAYLOADS = path.join(ROOT, 'payloads');
 const CHUNKS = path.join(ROOT, 'chunks');
 const REQUESTS = path.join(ROOT, 'requests');
 const preparing = new Map();
+const UPLOAD_RETRY_BASE_MS = 1_000;
+const UPLOAD_RETRY_MAX_MS = 60_000;
+
+function isProjectId(value) {
+  return require('./tree/identity').isUuid(String(value || ''));
+}
+
+function locatorChecksum(chunks) {
+  return manifestChecksum(chunks);
+}
 
-function sha256(bytes) {
-  return crypto.createHash('sha256').update(bytes).digest('hex');
+function locatorBytes(chunks) {
+  return chunks.reduce((total, chunk) => total + chunk.bytes, 0);
 }
 
 function ensureDirs() {
@@ -56,8 +71,7 @@ function readJson(file) {
 }
 
 function writeJson(file, value) {
-  fs.mkdirSync(path.dirname(file), { recursive: true });
-  fs.writeFileSync(file, `${JSON.stringify(value)}\n`, { mode: 0o600 });
+  durable.writeJson(file, value);
 }
 
 function publicLocator(manifest) {
@@ -83,14 +97,18 @@ function publicPayloadLocator(manifest) {
   };
 }
 
-function writeChunks(bytes) {
-  const chunks = [];
-  for (let offset = 0; offset < bytes.length; offset += CHUNK_BYTES) {
-    const part = bytes.subarray(offset, Math.min(offset + CHUNK_BYTES, bytes.length));
-    const digest = sha256(part);
-    const chunkFile = path.join(CHUNKS, `${digest}.bin`);
-    if (!fs.existsSync(chunkFile)) fs.writeFileSync(chunkFile, part, { mode: 0o600 });
-    chunks.push({ sha256: digest, bytes: part.length, uploaded: false });
+function describeChunks(bytes) {
+  return describeBytes(bytes).map((chunk) => ({ ...chunk, uploaded: false }));
+}
+
+function writeChunks(bytes, chunks = describeChunks(bytes)) {
+  let offset = 0;
+  for (const chunk of chunks) {
+    const part = bytes.subarray(offset, offset + chunk.bytes);
+    const chunkFile = path.join(CHUNKS, `${chunk.sha256}.bin`);
+    const held = fs.existsSync(chunkFile) ? fs.readFileSync(chunkFile) : null;
+    if (!held || sha256(held) !== chunk.sha256) durable.writeFile(chunkFile, part);
+    offset += chunk.bytes;
   }
   return chunks;
 }
@@ -113,8 +131,8 @@ async function prepareBundle(worktreeRoot, repoKey, resourceId, oid, file) {
       repoKey,
       oid,
       ref,
-      checksum: sha256(bundle),
-      bytes: bundle.length,
+      checksum: locatorChecksum(chunks),
+      bytes: locatorBytes(chunks),
       chunks,
       createdAt: new Date().toISOString(),
     });
@@ -128,23 +146,29 @@ function publishPayload(key, bytesInput, repoKey) {
   ensureDirs();
   const bytes = Buffer.isBuffer(bytesInput) ? bytesInput : Buffer.from(bytesInput);
   const project = require('./wall').projectForRepo(repoKey);
-  const checksum = sha256(bytes);
   if (!project?.resourceId) {
-    return { kind: 'pending', contract: PAYLOAD_CONTRACT, checksum };
+    return { kind: 'pending', contract: PAYLOAD_CONTRACT };
   }
+  const chunks = describeChunks(bytes);
+  const checksum = locatorChecksum(chunks);
   // Payload content is its identity. A later checkpoint that still contains
   // the same large file must reference the already-uploaded bytes instead of
   // creating another upload merely because the surrounding checkpoint moved.
   const file = payloadFile(project.resourceId, checksum);
   let manifest = readJson(file);
-  if (!manifest || manifest.checksum !== checksum) {
+  if (
+    !manifest
+    || manifest.contract !== PAYLOAD_CONTRACT
+    || manifest.checksum !== checksum
+  ) {
+    writeChunks(bytes, chunks);
     manifest = {
       contract: PAYLOAD_CONTRACT,
       resourceId: project.resourceId,
       repoKey,
       checksum,
-      bytes: bytes.length,
-      chunks: writeChunks(bytes),
+      bytes: locatorBytes(chunks),
+      chunks,
       createdAt: new Date().toISOString(),
     };
     writeJson(file, manifest);
@@ -182,18 +206,38 @@ async function bundleSourceFor(worktreeRoot, repoKey, oid) {
 
 async function objectSourceFor(worktreeRoot, repoKey, oid) {
   if (!oid || !repoKey) return null;
-  if (await originContains(worktreeRoot, oid)) return { kind: 'remote' };
   return bundleSourceFor(worktreeRoot, repoKey, oid);
 }
 
 function claimUpload() {
   ensureDirs();
+  const now = Date.now();
   for (const directory of [MANIFESTS, PAYLOADS]) {
     for (const name of fs.readdirSync(directory).sort()) {
-      const manifest = readJson(path.join(directory, name));
-      const chunk = manifest?.chunks?.find((entry) => !entry.uploaded);
+      const file = path.join(directory, name);
+      const manifest = readJson(file);
+      const chunk = manifest?.chunks?.find((entry) => (
+        !entry.uploaded
+        && !entry.uploadTerminal
+        && (
+          !entry.uploadRetryAfter
+          || !Number.isFinite(Date.parse(entry.uploadRetryAfter))
+          || Date.parse(entry.uploadRetryAfter) <= now
+        )
+      ));
       if (!chunk) continue;
-      const bytes = fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`));
+      const chunkFile = path.join(CHUNKS, `${chunk.sha256}.bin`);
+      const bytes = fs.existsSync(chunkFile) ? fs.readFileSync(chunkFile) : null;
+      if (
+        !bytes
+        || bytes.length !== chunk.bytes
+        || sha256(bytes) !== chunk.sha256
+      ) {
+        fs.rmSync(chunkFile, { force: true });
+        fs.rmSync(file, { force: true });
+        wake('delete', chunk.sha256);
+        continue;
+      }
       return {
         resourceId: manifest.resourceId,
         sha256: chunk.sha256,
@@ -219,6 +263,10 @@ function acknowledgeUpload(resourceId, digest) {
       for (const chunk of manifest.chunks) {
         if (chunk.sha256 === digest && !chunk.uploaded) {
           chunk.uploaded = true;
+          delete chunk.uploadAttempts;
+          delete chunk.uploadLastError;
+          delete chunk.uploadRetryAfter;
+          delete chunk.uploadTerminal;
           touched = true;
           changed = true;
         }
@@ -239,21 +287,73 @@ function acknowledgeUpload(resourceId, digest) {
   return { changed, completed };
 }
 
+function failUpload(resourceId, digest, failure = {}) {
+  if (!isProjectId(resourceId) || !/^[0-9a-f]{64}$/.test(String(digest || ''))) {
+    return { changed: false, terminal: false };
+  }
+  ensureDirs();
+  const failedAt = new Date().toISOString();
+  const retryable = failure?.retryable !== false;
+  let changed = false;
+  let terminal = false;
+  let retryAfter = null;
+  let attempts = 0;
+  for (const directory of [MANIFESTS, PAYLOADS]) {
+    for (const name of fs.readdirSync(directory)) {
+      const file = path.join(directory, name);
+      const manifest = readJson(file);
+      if (manifest?.resourceId !== resourceId || !manifest?.chunks) continue;
+      const chunk = manifest.chunks.find((entry) => (
+        entry.sha256 === digest && !entry.uploaded
+      ));
+      if (!chunk) continue;
+      attempts = Math.max(attempts, Number(chunk.uploadAttempts || 0) + 1);
+      chunk.uploadAttempts = attempts;
+      chunk.uploadLastError = {
+        code: typeof failure?.code === 'string' ? failure.code : 'repo_object_upload_failed',
+        message: typeof failure?.message === 'string' ? failure.message : 'repo object upload failed',
+        retryable,
+        at: failedAt,
+      };
+      if (retryable) {
+        const delayMs = Math.min(
+          UPLOAD_RETRY_MAX_MS,
+          UPLOAD_RETRY_BASE_MS * (2 ** Math.min(attempts - 1, 16)),
+        );
+        retryAfter = new Date(Date.now() + delayMs).toISOString();
+        chunk.uploadRetryAfter = retryAfter;
+        delete chunk.uploadTerminal;
+      } else {
+        chunk.uploadTerminal = { ...chunk.uploadLastError };
+        delete chunk.uploadRetryAfter;
+        terminal = true;
+      }
+      writeJson(file, manifest);
+      changed = true;
+    }
+  }
+  if (changed) wake('update', digest);
+  return { changed, terminal, retryAfter, attempts };
+}
+
 function validatePayloadLocator(locator) {
   if (
     locator?.kind !== 'chunks'
     || locator?.contract !== PAYLOAD_CONTRACT
-    || !/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(locator.resourceId || ''))
+    || !isProjectId(locator.resourceId)
     || !/^[0-9a-f]{64}$/.test(String(locator.checksum || ''))
     || !Number.isSafeInteger(locator.bytes)
-    || locator.bytes < 0
+    || locator.bytes < 1
     || !Array.isArray(locator.chunks)
+    || locator.chunks.length < 1
     || locator.chunks.some((chunk) => (
       !/^[0-9a-f]{64}$/.test(String(chunk?.sha256 || ''))
       || !Number.isSafeInteger(chunk?.bytes)
-      || chunk.bytes < 0
+      || chunk.bytes < 1
       || chunk.bytes > CHUNK_BYTES
     ))
+    || locatorBytes(locator.chunks) !== locator.bytes
+    || locatorChecksum(locator.chunks) !== locator.checksum
   ) throw new Error('invalid checkpoint payload locator');
 }
 
@@ -263,7 +363,11 @@ function readPayload(locator) {
   let waiting = false;
   for (const chunk of locator.chunks) {
     const file = path.join(CHUNKS, `${chunk.sha256}.bin`);
-    if (fs.existsSync(file) && fs.statSync(file).size === chunk.bytes) continue;
+    if (fs.existsSync(file)) {
+      const held = fs.readFileSync(file);
+      if (held.length === chunk.bytes && sha256(held) === chunk.sha256) continue;
+      fs.rmSync(file, { force: true });
+    }
     writeJson(requestFile(locator.resourceId, chunk.sha256), {
       resourceId: locator.resourceId,
       sha256: chunk.sha256,
@@ -282,7 +386,7 @@ function readPayload(locator) {
     return bytes;
   });
   const bytes = Buffer.concat(parts);
-  if (bytes.length !== locator.bytes || sha256(bytes) !== locator.checksum) {
+  if (bytes.length !== locator.bytes) {
     throw new Error('checkpoint payload verification failed');
   }
   writeJson(payloadFile(locator.resourceId, locator.checksum), {
@@ -303,21 +407,29 @@ function requestFile(resourceId, digest) {
 
 function claimDownload() {
   ensureDirs();
-  const name = fs.readdirSync(REQUESTS).filter((entry) => entry.endsWith('.json')).sort()[0];
-  if (!name) return null;
-  const request = readJson(path.join(REQUESTS, name));
-  return request?.sha256 ? request : null;
+  for (const name of fs.readdirSync(REQUESTS).filter((entry) => entry.endsWith('.json')).sort()) {
+    const file = path.join(REQUESTS, name);
+    const request = readJson(file);
+    if (
+      request
+      && Object.keys(request).length === 2
+      && isProjectId(request.resourceId)
+      && /^[0-9a-f]{64}$/.test(String(request.sha256 || ''))
+    ) return request;
+    fs.rmSync(file, { force: true });
+  }
+  return null;
 }
 
 function receiveDownload(resourceId, digest, dataBase64) {
-  if (!/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(resourceId || ''))) {
+  if (!isProjectId(resourceId)) {
     throw new Error('invalid repo object project');
   }
   if (!/^[0-9a-f]{64}$/.test(String(digest || ''))) throw new Error('invalid repo object checksum');
   ensureDirs();
   const bytes = Buffer.from(String(dataBase64 || ''), 'base64');
   if (sha256(bytes) !== digest) throw new Error('repo object checksum mismatch');
-  fs.writeFileSync(path.join(CHUNKS, `${digest}.bin`), bytes, { mode: 0o600 });
+  durable.writeFile(path.join(CHUNKS, `${digest}.bin`), bytes);
   fs.rmSync(requestFile(resourceId, digest), { force: true });
   wake('update', digest);
   require('./reducer').reduceNow().catch(() => {});
@@ -326,17 +438,24 @@ function receiveDownload(resourceId, digest, dataBase64) {
 
 function validateLocator(locator) {
   if (
-    locator?.contract !== CONTRACT
-    || !/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(locator.resourceId || ''))
-    || !/^[0-9a-f]{40,64}$/.test(String(locator.oid || ''))
+    locator?.kind !== 'bundle'
+    || locator?.contract !== CONTRACT
+    || !isProjectId(locator.resourceId)
+    || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(String(locator.oid || ''))
     || !/^[0-9a-f]{64}$/.test(String(locator.checksum || ''))
+    || locator.ref !== `refs/amalgm/exports/${locator.oid}`
+    || !Number.isSafeInteger(locator.bytes)
+    || locator.bytes < 1
     || !Array.isArray(locator.chunks)
+    || locator.chunks.length < 1
     || locator.chunks.some((chunk) => (
       !/^[0-9a-f]{64}$/.test(String(chunk?.sha256 || ''))
       || !Number.isSafeInteger(chunk?.bytes)
-      || chunk.bytes < 0
+      || chunk.bytes < 1
       || chunk.bytes > CHUNK_BYTES
     ))
+    || locatorBytes(locator.chunks) !== locator.bytes
+    || locatorChecksum(locator.chunks) !== locator.checksum
   ) throw new Error('invalid git bundle locator');
 }
 
@@ -346,7 +465,11 @@ function prepareBundleImport(locator) {
   let waiting = false;
   for (const chunk of locator.chunks) {
     const file = path.join(CHUNKS, `${chunk.sha256}.bin`);
-    if (fs.existsSync(file) && fs.statSync(file).size === chunk.bytes) continue;
+    if (fs.existsSync(file)) {
+      const held = fs.readFileSync(file);
+      if (held.length === chunk.bytes && sha256(held) === chunk.sha256) continue;
+      fs.rmSync(file, { force: true });
+    }
     writeJson(requestFile(locator.resourceId, chunk.sha256), {
       resourceId: locator.resourceId,
       sha256: chunk.sha256,
@@ -359,19 +482,24 @@ function prepareBundleImport(locator) {
   }
 
   const bundleFile = path.join(ROOT, `${locator.checksum}.bundle`);
-  if (!fs.existsSync(bundleFile)) {
-    const parts = locator.chunks.map((chunk) => {
-      const bytes = fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`));
-      if (bytes.length !== chunk.bytes || sha256(bytes) !== chunk.sha256) {
-        throw new Error('git bundle chunk verification failed');
-      }
-      return bytes;
-    });
-    const bundle = Buffer.concat(parts);
-    if (bundle.length !== locator.bytes || sha256(bundle) !== locator.checksum) {
-      throw new Error('git bundle verification failed');
+  const parts = locator.chunks.map((chunk) => (
+    fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`))
+  ));
+  const bundleMatches = fs.existsSync(bundleFile) && (() => {
+    const held = fs.readFileSync(bundleFile);
+    let offset = 0;
+    for (let index = 0; index < locator.chunks.length; index += 1) {
+      const chunk = locator.chunks[index];
+      const part = held.subarray(offset, offset + chunk.bytes);
+      if (part.length !== chunk.bytes || sha256(part) !== chunk.sha256) return false;
+      offset += chunk.bytes;
     }
-    fs.writeFileSync(bundleFile, bundle, { mode: 0o600 });
+    return offset === held.length && held.length === locator.bytes;
+  })();
+  if (!bundleMatches) {
+    const bundle = Buffer.concat(parts);
+    if (bundle.length !== locator.bytes) throw new Error('git bundle verification failed');
+    durable.writeFile(bundleFile, bundle);
   }
   return { state: 'ready', file: bundleFile };
 }
@@ -387,10 +515,12 @@ module.exports = {
   CHUNK_BYTES,
   CONTRACT,
   PAYLOAD_CONTRACT,
+  _test: { describeChunks, locatorBytes, locatorChecksum },
   acknowledgeUpload,
   bundleSourceFor,
   claimDownload,
   claimUpload,
+  failUpload,
   importFromLocator,
   objectSourceFor,
   prepareBundleImport,