amalgm 0.1.245 → 0.1.246

package/runtime/scripts/amalgm-mcp/workspace/objects.js

@@ -13,44 +13,22 @@ const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
 const { AMALGM_DIR } = require('../config');
-const { isSharedResourceId } = require('../state/cloud-resource-id');
 const { appendStateEvent } = require('../state/events');
-const durable = require('./durable');
+const { originContains } = require('./git');
 const transition = require('./transition');
-const {
-  CHUNK_BYTES,
-  describeBytes,
-  manifestChecksum,
-  sha256,
-} = require('./content');
 
-const CONTRACT = 'git-bundle-chunks@2';
-const PAYLOAD_CONTRACT = 'opaque-chunks@2';
+const CONTRACT = 'git-bundle-chunks@1';
+const PAYLOAD_CONTRACT = 'opaque-chunks@1';
+const CHUNK_BYTES = 512 * 1024;
 const ROOT = path.join(AMALGM_DIR, 'state', 'repo-objects');
 const MANIFESTS = path.join(ROOT, 'manifests');
 const PAYLOADS = path.join(ROOT, 'payloads');
 const CHUNKS = path.join(ROOT, 'chunks');
 const REQUESTS = path.join(ROOT, 'requests');
 const preparing = new Map();
-const UPLOAD_RETRY_BASE_MS = 1_000;
-const UPLOAD_RETRY_MAX_MS = 60_000;
-const RESOURCE_WIDE_UPLOAD_FAILURES = new Set([
-  'invalid_repo_project',
-  'private_resource_forbidden',
-  'shared_resource_read_forbidden',
-  'shared_resource_write_forbidden',
-]);
-
-function isProjectId(value) {
-  return require('./tree/identity').isUuid(String(value || ''));
-}
-
-function locatorChecksum(chunks) {
-  return manifestChecksum(chunks);
-}
 
-function locatorBytes(chunks) {
-  return chunks.reduce((total, chunk) => total + chunk.bytes, 0);
+function sha256(bytes) {
+  return crypto.createHash('sha256').update(bytes).digest('hex');
 }
 
 function ensureDirs() {
@@ -78,7 +56,8 @@ function readJson(file) {
 }
 
 function writeJson(file, value) {
-  durable.writeJson(file, value);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, `${JSON.stringify(value)}\n`, { mode: 0o600 });
 }
 
 function publicLocator(manifest) {
@@ -104,18 +83,14 @@ function publicPayloadLocator(manifest) {
   };
 }
 
-function describeChunks(bytes) {
-  return describeBytes(bytes).map((chunk) => ({ ...chunk, uploaded: false }));
-}
-
-function writeChunks(bytes, chunks = describeChunks(bytes)) {
-  let offset = 0;
-  for (const chunk of chunks) {
-    const part = bytes.subarray(offset, offset + chunk.bytes);
-    const chunkFile = path.join(CHUNKS, `${chunk.sha256}.bin`);
-    const held = fs.existsSync(chunkFile) ? fs.readFileSync(chunkFile) : null;
-    if (!held || sha256(held) !== chunk.sha256) durable.writeFile(chunkFile, part);
-    offset += chunk.bytes;
+function writeChunks(bytes) {
+  const chunks = [];
+  for (let offset = 0; offset < bytes.length; offset += CHUNK_BYTES) {
+    const part = bytes.subarray(offset, Math.min(offset + CHUNK_BYTES, bytes.length));
+    const digest = sha256(part);
+    const chunkFile = path.join(CHUNKS, `${digest}.bin`);
+    if (!fs.existsSync(chunkFile)) fs.writeFileSync(chunkFile, part, { mode: 0o600 });
+    chunks.push({ sha256: digest, bytes: part.length, uploaded: false });
   }
   return chunks;
 }
@@ -138,8 +113,8 @@ async function prepareBundle(worktreeRoot, repoKey, resourceId, oid, file) {
       repoKey,
       oid,
       ref,
-      checksum: locatorChecksum(chunks),
-      bytes: locatorBytes(chunks),
+      checksum: sha256(bundle),
+      bytes: bundle.length,
       chunks,
       createdAt: new Date().toISOString(),
     });
@@ -153,29 +128,20 @@ function publishPayload(key, bytesInput, repoKey) {
   ensureDirs();
   const bytes = Buffer.isBuffer(bytesInput) ? bytesInput : Buffer.from(bytesInput);
   const project = require('./wall').projectForRepo(repoKey);
+  const checksum = sha256(bytes);
   if (!project?.resourceId) {
-    return { kind: 'pending', contract: PAYLOAD_CONTRACT };
+    return { kind: 'pending', contract: PAYLOAD_CONTRACT, checksum };
   }
-  const chunks = describeChunks(bytes);
-  const checksum = locatorChecksum(chunks);
-  // Payload content is its identity. A later checkpoint that still contains
-  // the same large file must reference the already-uploaded bytes instead of
-  // creating another upload merely because the surrounding checkpoint moved.
-  const file = payloadFile(project.resourceId, checksum);
+  const file = payloadFile(project.resourceId, key);
   let manifest = readJson(file);
-  if (
-    !manifest
-    || manifest.contract !== PAYLOAD_CONTRACT
-    || manifest.checksum !== checksum
-  ) {
-    writeChunks(bytes, chunks);
+  if (!manifest || manifest.checksum !== checksum) {
     manifest = {
       contract: PAYLOAD_CONTRACT,
       resourceId: project.resourceId,
       repoKey,
       checksum,
-      bytes: locatorBytes(chunks),
-      chunks,
+      bytes: bytes.length,
+      chunks: writeChunks(bytes),
       createdAt: new Date().toISOString(),
     };
     writeJson(file, manifest);
@@ -213,38 +179,18 @@ async function bundleSourceFor(worktreeRoot, repoKey, oid) {
 
 async function objectSourceFor(worktreeRoot, repoKey, oid) {
   if (!oid || !repoKey) return null;
+  if (await originContains(worktreeRoot, oid)) return { kind: 'remote' };
   return bundleSourceFor(worktreeRoot, repoKey, oid);
 }
 
 function claimUpload() {
   ensureDirs();
-  const now = Date.now();
   for (const directory of [MANIFESTS, PAYLOADS]) {
     for (const name of fs.readdirSync(directory).sort()) {
-      const file = path.join(directory, name);
-      const manifest = readJson(file);
-      const chunk = manifest?.chunks?.find((entry) => (
-        !entry.uploaded
-        && !entry.uploadTerminal
-        && (
-          !entry.uploadRetryAfter
-          || !Number.isFinite(Date.parse(entry.uploadRetryAfter))
-          || Date.parse(entry.uploadRetryAfter) <= now
-        )
-      ));
+      const manifest = readJson(path.join(directory, name));
+      const chunk = manifest?.chunks?.find((entry) => !entry.uploaded);
       if (!chunk) continue;
-      const chunkFile = path.join(CHUNKS, `${chunk.sha256}.bin`);
-      const bytes = fs.existsSync(chunkFile) ? fs.readFileSync(chunkFile) : null;
-      if (
-        !bytes
-        || bytes.length !== chunk.bytes
-        || sha256(bytes) !== chunk.sha256
-      ) {
-        fs.rmSync(chunkFile, { force: true });
-        fs.rmSync(file, { force: true });
-        wake('delete', chunk.sha256);
-        continue;
-      }
+      const bytes = fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`));
       return {
         resourceId: manifest.resourceId,
         sha256: chunk.sha256,
@@ -270,10 +216,6 @@ function acknowledgeUpload(resourceId, digest) {
       for (const chunk of manifest.chunks) {
         if (chunk.sha256 === digest && !chunk.uploaded) {
           chunk.uploaded = true;
-          delete chunk.uploadAttempts;
-          delete chunk.uploadLastError;
-          delete chunk.uploadRetryAfter;
-          delete chunk.uploadTerminal;
           touched = true;
           changed = true;
         }
@@ -294,89 +236,21 @@ function acknowledgeUpload(resourceId, digest) {
   return { changed, completed };
 }
 
-function failUpload(resourceId, digest, failure = {}) {
-  // Upload work is durable across protocol generations. Match the exact
-  // identity that was accepted when the work was queued, including legacy
-  // `shr_` rails, so stale historical work can reach a terminal state instead
-  // of monopolizing every newer code project behind it.
-  if (!isSharedResourceId(resourceId) || !/^[0-9a-f]{64}$/.test(String(digest || ''))) {
-    return { changed: false, terminal: false };
-  }
-  ensureDirs();
-  const failedAt = new Date().toISOString();
-  const retryable = failure?.retryable !== false;
-  const failureCode = typeof failure?.code === 'string'
-    ? failure.code
-    : 'repo_object_upload_failed';
-  const resourceWide = RESOURCE_WIDE_UPLOAD_FAILURES.has(failureCode);
-  let changed = false;
-  let terminal = false;
-  let retryAfter = null;
-  let attempts = 0;
-  for (const directory of [MANIFESTS, PAYLOADS]) {
-    for (const name of fs.readdirSync(directory)) {
-      const file = path.join(directory, name);
-      const manifest = readJson(file);
-      if (manifest?.resourceId !== resourceId || !manifest?.chunks) continue;
-      const failedChunks = manifest.chunks.filter((entry) => (
-        !entry.uploaded && (resourceWide || entry.sha256 === digest)
-      ));
-      if (failedChunks.length < 1) continue;
-      for (const chunk of failedChunks) {
-        const chunkAttempts = Number(chunk.uploadAttempts || 0) + 1;
-        attempts = Math.max(attempts, chunkAttempts);
-        chunk.uploadAttempts = chunkAttempts;
-        chunk.uploadLastError = {
-          code: failureCode,
-          message: typeof failure?.message === 'string'
-            ? failure.message
-            : 'repo object upload failed',
-          retryable,
-          at: failedAt,
-        };
-        if (retryable) {
-          const delayMs = Math.min(
-            UPLOAD_RETRY_MAX_MS,
-            UPLOAD_RETRY_BASE_MS * (2 ** Math.min(chunkAttempts - 1, 16)),
-          );
-          const chunkRetryAfter = new Date(Date.now() + delayMs).toISOString();
-          retryAfter = retryAfter && Date.parse(retryAfter) > Date.parse(chunkRetryAfter)
-            ? retryAfter
-            : chunkRetryAfter;
-          chunk.uploadRetryAfter = chunkRetryAfter;
-          delete chunk.uploadTerminal;
-        } else {
-          chunk.uploadTerminal = { ...chunk.uploadLastError };
-          delete chunk.uploadRetryAfter;
-          terminal = true;
-        }
-      }
-      writeJson(file, manifest);
-      changed = true;
-    }
-  }
-  if (changed) wake('update', digest);
-  return { changed, terminal, retryAfter, attempts };
-}
-
 function validatePayloadLocator(locator) {
   if (
     locator?.kind !== 'chunks'
     || locator?.contract !== PAYLOAD_CONTRACT
-    || !isProjectId(locator.resourceId)
+    || !/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(locator.resourceId || ''))
     || !/^[0-9a-f]{64}$/.test(String(locator.checksum || ''))
     || !Number.isSafeInteger(locator.bytes)
-    || locator.bytes < 1
+    || locator.bytes < 0
     || !Array.isArray(locator.chunks)
-    || locator.chunks.length < 1
     || locator.chunks.some((chunk) => (
       !/^[0-9a-f]{64}$/.test(String(chunk?.sha256 || ''))
       || !Number.isSafeInteger(chunk?.bytes)
-      || chunk.bytes < 1
+      || chunk.bytes < 0
       || chunk.bytes > CHUNK_BYTES
     ))
-    || locatorBytes(locator.chunks) !== locator.bytes
-    || locatorChecksum(locator.chunks) !== locator.checksum
   ) throw new Error('invalid checkpoint payload locator');
 }
 
@@ -386,11 +260,7 @@ function readPayload(locator) {
   let waiting = false;
   for (const chunk of locator.chunks) {
     const file = path.join(CHUNKS, `${chunk.sha256}.bin`);
-    if (fs.existsSync(file)) {
-      const held = fs.readFileSync(file);
-      if (held.length === chunk.bytes && sha256(held) === chunk.sha256) continue;
-      fs.rmSync(file, { force: true });
-    }
+    if (fs.existsSync(file) && fs.statSync(file).size === chunk.bytes) continue;
     writeJson(requestFile(locator.resourceId, chunk.sha256), {
       resourceId: locator.resourceId,
       sha256: chunk.sha256,
@@ -409,18 +279,9 @@ function readPayload(locator) {
     return bytes;
   });
   const bytes = Buffer.concat(parts);
-  if (bytes.length !== locator.bytes) {
+  if (bytes.length !== locator.bytes || sha256(bytes) !== locator.checksum) {
     throw new Error('checkpoint payload verification failed');
   }
-  writeJson(payloadFile(locator.resourceId, locator.checksum), {
-    contract: PAYLOAD_CONTRACT,
-    resourceId: locator.resourceId,
-    repoKey: null,
-    checksum: locator.checksum,
-    bytes: locator.bytes,
-    chunks: locator.chunks.map((chunk) => ({ ...chunk, uploaded: true })),
-    createdAt: new Date().toISOString(),
-  });
   return { state: 'ready', bytes };
 }
 
@@ -430,29 +291,21 @@ function requestFile(resourceId, digest) {
 
 function claimDownload() {
   ensureDirs();
-  for (const name of fs.readdirSync(REQUESTS).filter((entry) => entry.endsWith('.json')).sort()) {
-    const file = path.join(REQUESTS, name);
-    const request = readJson(file);
-    if (
-      request
-      && Object.keys(request).length === 2
-      && isProjectId(request.resourceId)
-      && /^[0-9a-f]{64}$/.test(String(request.sha256 || ''))
-    ) return request;
-    fs.rmSync(file, { force: true });
-  }
-  return null;
+  const name = fs.readdirSync(REQUESTS).filter((entry) => entry.endsWith('.json')).sort()[0];
+  if (!name) return null;
+  const request = readJson(path.join(REQUESTS, name));
+  return request?.sha256 ? request : null;
 }
 
 function receiveDownload(resourceId, digest, dataBase64) {
-  if (!isProjectId(resourceId)) {
+  if (!/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(resourceId || ''))) {
     throw new Error('invalid repo object project');
   }
   if (!/^[0-9a-f]{64}$/.test(String(digest || ''))) throw new Error('invalid repo object checksum');
   ensureDirs();
   const bytes = Buffer.from(String(dataBase64 || ''), 'base64');
   if (sha256(bytes) !== digest) throw new Error('repo object checksum mismatch');
-  durable.writeFile(path.join(CHUNKS, `${digest}.bin`), bytes);
+  fs.writeFileSync(path.join(CHUNKS, `${digest}.bin`), bytes, { mode: 0o600 });
   fs.rmSync(requestFile(resourceId, digest), { force: true });
   wake('update', digest);
   require('./reducer').reduceNow().catch(() => {});
@@ -461,24 +314,17 @@ function receiveDownload(resourceId, digest, dataBase64) {
 
 function validateLocator(locator) {
   if (
-    locator?.kind !== 'bundle'
-    || locator?.contract !== CONTRACT
-    || !isProjectId(locator.resourceId)
-    || !/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(String(locator.oid || ''))
+    locator?.contract !== CONTRACT
+    || !/^shr_[A-Za-z0-9_-]{16,128}$/.test(String(locator.resourceId || ''))
+    || !/^[0-9a-f]{40,64}$/.test(String(locator.oid || ''))
     || !/^[0-9a-f]{64}$/.test(String(locator.checksum || ''))
-    || locator.ref !== `refs/amalgm/exports/${locator.oid}`
-    || !Number.isSafeInteger(locator.bytes)
-    || locator.bytes < 1
     || !Array.isArray(locator.chunks)
-    || locator.chunks.length < 1
     || locator.chunks.some((chunk) => (
       !/^[0-9a-f]{64}$/.test(String(chunk?.sha256 || ''))
       || !Number.isSafeInteger(chunk?.bytes)
-      || chunk.bytes < 1
+      || chunk.bytes < 0
       || chunk.bytes > CHUNK_BYTES
     ))
-    || locatorBytes(locator.chunks) !== locator.bytes
-    || locatorChecksum(locator.chunks) !== locator.checksum
   ) throw new Error('invalid git bundle locator');
 }
 
@@ -488,11 +334,7 @@ function prepareBundleImport(locator) {
   let waiting = false;
   for (const chunk of locator.chunks) {
     const file = path.join(CHUNKS, `${chunk.sha256}.bin`);
-    if (fs.existsSync(file)) {
-      const held = fs.readFileSync(file);
-      if (held.length === chunk.bytes && sha256(held) === chunk.sha256) continue;
-      fs.rmSync(file, { force: true });
-    }
+    if (fs.existsSync(file) && fs.statSync(file).size === chunk.bytes) continue;
     writeJson(requestFile(locator.resourceId, chunk.sha256), {
       resourceId: locator.resourceId,
       sha256: chunk.sha256,
@@ -505,24 +347,19 @@ function prepareBundleImport(locator) {
   }
 
   const bundleFile = path.join(ROOT, `${locator.checksum}.bundle`);
-  const parts = locator.chunks.map((chunk) => (
-    fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`))
-  ));
-  const bundleMatches = fs.existsSync(bundleFile) && (() => {
-    const held = fs.readFileSync(bundleFile);
-    let offset = 0;
-    for (let index = 0; index < locator.chunks.length; index += 1) {
-      const chunk = locator.chunks[index];
-      const part = held.subarray(offset, offset + chunk.bytes);
-      if (part.length !== chunk.bytes || sha256(part) !== chunk.sha256) return false;
-      offset += chunk.bytes;
-    }
-    return offset === held.length && held.length === locator.bytes;
-  })();
-  if (!bundleMatches) {
+  if (!fs.existsSync(bundleFile)) {
+    const parts = locator.chunks.map((chunk) => {
+      const bytes = fs.readFileSync(path.join(CHUNKS, `${chunk.sha256}.bin`));
+      if (bytes.length !== chunk.bytes || sha256(bytes) !== chunk.sha256) {
+        throw new Error('git bundle chunk verification failed');
+      }
+      return bytes;
+    });
     const bundle = Buffer.concat(parts);
-    if (bundle.length !== locator.bytes) throw new Error('git bundle verification failed');
-    durable.writeFile(bundleFile, bundle);
+    if (bundle.length !== locator.bytes || sha256(bundle) !== locator.checksum) {
+      throw new Error('git bundle verification failed');
+    }
+    fs.writeFileSync(bundleFile, bundle, { mode: 0o600 });
   }
   return { state: 'ready', file: bundleFile };
 }
@@ -538,12 +375,10 @@ module.exports = {
   CHUNK_BYTES,
   CONTRACT,
   PAYLOAD_CONTRACT,
-  _test: { describeChunks, locatorBytes, locatorChecksum },
   acknowledgeUpload,
   bundleSourceFor,
   claimDownload,
   claimUpload,
-  failUpload,
   importFromLocator,
   objectSourceFor,
   prepareBundleImport,