amalgm 0.1.245 → 0.1.246

package/runtime/scripts/amalgm-mcp/registration/entity-content.js

@@ -0,0 +1,345 @@
+'use strict';
+
+/**
+ * Immutable local cache and upload ledger for file entity content.
+ *
+ * The entity registry owns UUID facts. This module owns only the bytes named
+ * by a file record's payloadVersion: capture the exact observed bytes once,
+ * split them into bounded content-addressed chunks, and retain a durable
+ * upload cursor until the private cloud resource acknowledges the manifest.
+ * It deliberately knows no local entity path or watcher binding.
+ */
+
+const crypto = require('crypto');
+const fs = require('fs');
+const path = require('path');
+
+const CONTENT_CONTRACT = 'amalgm-entity-content@1';
+const CHUNK_BYTES = 256 * 1024;
+const HASH = /^[0-9a-f]{64}$/;
+
+function digest(bytes) {
+  return crypto.createHash('sha256').update(bytes).digest('hex');
+}
+
+function now() {
+  return new Date().toISOString();
+}
+
+function assertHash(value, name = 'content hash') {
+  if (typeof value !== 'string' || !HASH.test(value)) throw new Error(`invalid ${name}`);
+  return value;
+}
+
+function safeName(value) {
+  return crypto.createHash('sha256').update(String(value)).digest('hex');
+}
+
+function createEntityContent({ dir }) {
+  if (typeof dir !== 'string' || !dir) throw new Error('createEntityContent requires an entity directory');
+  const root = path.join(dir, 'content');
+  const chunksDir = path.join(root, 'chunks');
+  const manifestsDir = path.join(root, 'manifests');
+  const uploadsDir = path.join(root, 'uploads');
+
+  function ensureDirs() {
+    for (const target of [chunksDir, manifestsDir, uploadsDir]) {
+      fs.mkdirSync(target, { recursive: true, mode: 0o700 });
+    }
+  }
+
+  function chunkFile(checksum) {
+    return path.join(chunksDir, `${assertHash(checksum, 'chunk checksum')}.bin`);
+  }
+
+  function manifestFile(contentHash) {
+    return path.join(manifestsDir, `${assertHash(contentHash)}.json`);
+  }
+
+  function uploadFile(resourceId, contentHash) {
+    assertHash(contentHash);
+    return path.join(uploadsDir, safeName(resourceId), `${contentHash}.json`);
+  }
+
+  function readJson(file) {
+    try {
+      return JSON.parse(fs.readFileSync(file, 'utf8'));
+    } catch {
+      return null;
+    }
+  }
+
+  function writeJson(file, value) {
+    fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+    const temporary = `${file}.${crypto.randomBytes(12).toString('hex')}.tmp`;
+    try {
+      fs.writeFileSync(temporary, `${JSON.stringify(value)}\n`, { mode: 0o600, flag: 'wx' });
+      // Every writer for this file derived the same manifest from the same
+      // content hash. Atomic replacement is therefore safe across local
+      // processes and works on filesystems that do not support hard links.
+      fs.renameSync(temporary, file);
+    } finally {
+      fs.rmSync(temporary, { force: true });
+    }
+  }
+
+  function validateManifest(value, expectedHash = null) {
+    if (!value || value.contract !== CONTENT_CONTRACT
+      || typeof value.contentHash !== 'string' || !HASH.test(value.contentHash)
+      || (expectedHash !== null && value.contentHash !== expectedHash)
+      || !Number.isSafeInteger(value.bytes) || value.bytes < 0
+      || !Array.isArray(value.chunks)
+      || value.chunks.some((chunk) => (
+        !chunk || !HASH.test(String(chunk.sha256 || ''))
+        || !Number.isSafeInteger(chunk.bytes) || chunk.bytes < 1 || chunk.bytes > CHUNK_BYTES
+      ))) {
+      throw new Error('entity content manifest is invalid');
+    }
+    if (value.chunks.reduce((total, chunk) => total + chunk.bytes, 0) !== value.bytes) {
+      throw new Error('entity content manifest byte count is invalid');
+    }
+    if (value.bytes === 0 && value.chunks.length !== 0) {
+      throw new Error('empty entity content must have no chunks');
+    }
+    return {
+      contract: CONTENT_CONTRACT,
+      contentHash: value.contentHash,
+      bytes: value.bytes,
+      chunks: value.chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
+    };
+  }
+
+  function manifest(contentHash) {
+    ensureDirs();
+    const value = readJson(manifestFile(contentHash));
+    return value ? validateManifest(value, contentHash) : null;
+  }
+
+  function writeChunk(checksum, bytes) {
+    const file = chunkFile(checksum);
+    try {
+      fs.writeFileSync(file, bytes, { mode: 0o600, flag: 'wx' });
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      const existing = fs.readFileSync(file);
+      if (existing.length !== bytes.length || digest(existing) !== checksum) {
+        throw new Error(`local content chunk ${checksum} is corrupt`);
+      }
+    }
+  }
+
+  /** Capture exactly the bytes that produced `contentHash`, never a later
+   * version of the file. A changed source is a retryable refusal: no
+   * manifest exists for the wrong head, and the watcher will observe again. */
+  function captureFile({ filePath, contentHash }) {
+    assertHash(contentHash);
+    if (typeof filePath !== 'string' || !path.isAbsolute(filePath)) {
+      throw new Error('entity content capture requires an absolute file path');
+    }
+    ensureDirs();
+    const existing = manifest(contentHash);
+    if (existing) return existing;
+
+    const full = crypto.createHash('sha256');
+    const chunks = [];
+    let bytes = 0;
+    const fd = fs.openSync(filePath, 'r');
+    try {
+      const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
+      let read;
+      while ((read = fs.readSync(fd, buffer, 0, buffer.length)) > 0) {
+        const part = Buffer.from(buffer.subarray(0, read));
+        const checksum = digest(part);
+        full.update(part);
+        writeChunk(checksum, part);
+        chunks.push({ sha256: checksum, bytes: part.length });
+        bytes += part.length;
+      }
+    } finally {
+      fs.closeSync(fd);
+    }
+    if (full.digest('hex') !== contentHash) {
+      throw new Error('file changed while its content was being captured');
+    }
+    const captured = { contract: CONTENT_CONTRACT, contentHash, bytes, chunks };
+    writeJson(manifestFile(contentHash), captured);
+    const installed = manifest(contentHash);
+    if (!installed) throw new Error('captured entity content manifest did not persist');
+    return installed;
+  }
+
+  function initialUpload(resourceId, contentHash) {
+    return {
+      contract: CONTENT_CONTRACT,
+      resourceId,
+      contentHash,
+      nextPart: 0,
+      manifestUploaded: false,
+      leaseUntil: null,
+      updatedAt: now(),
+    };
+  }
+
+  function validateUpload(value, resourceId, contentHash, content) {
+    if (!value || value.contract !== CONTENT_CONTRACT
+      || value.resourceId !== resourceId || value.contentHash !== contentHash
+      || !Number.isSafeInteger(value.nextPart) || value.nextPart < 0 || value.nextPart > content.chunks.length
+      || typeof value.manifestUploaded !== 'boolean') {
+      throw new Error('entity content upload ledger is invalid');
+    }
+    return value;
+  }
+
+  function readUpload(resourceId, contentHash, content) {
+    const file = uploadFile(resourceId, contentHash);
+    const value = readJson(file);
+    return value ? validateUpload(value, resourceId, contentHash, content) : null;
+  }
+
+  function saveUpload(value) {
+    const file = uploadFile(value.resourceId, value.contentHash);
+    fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+    const temporary = `${file}.${crypto.randomBytes(12).toString('hex')}.tmp`;
+    try {
+      fs.writeFileSync(temporary, `${JSON.stringify(value)}\n`, { mode: 0o600, flag: 'wx' });
+      fs.renameSync(temporary, file);
+    } finally {
+      fs.rmSync(temporary, { force: true });
+    }
+  }
+
+  function ensureUpload({ resourceId, contentHash }) {
+    if (typeof resourceId !== 'string' || !resourceId) throw new Error('entity content upload requires a resource id');
+    const content = manifest(contentHash);
+    if (!content) throw new Error(`content ${contentHash} was not captured before its entity record committed`);
+    const current = readUpload(resourceId, contentHash, content);
+    if (current) return current;
+    const created = initialUpload(resourceId, contentHash);
+    saveUpload(created);
+    return created;
+  }
+
+  function queueRecords({ resourceId, records }) {
+    if (!Array.isArray(records)) throw new Error('entity content queue requires records');
+    for (const record of records) {
+      if (!['file.text', 'file.binary'].includes(record?.type) || record.payloadVersion === null) continue;
+      ensureUpload({ resourceId, contentHash: record.payloadVersion });
+    }
+  }
+
+  function uploadEntries(resourceId) {
+    const directory = path.join(uploadsDir, safeName(resourceId));
+    try {
+      return fs.readdirSync(directory).filter((name) => name.endsWith('.json')).sort()
+        .map((name) => name.slice(0, -'.json'.length));
+    } catch (error) {
+      if (error.code === 'ENOENT') return [];
+      throw error;
+    }
+  }
+
+  function claim({ resourceId, leaseMs = 30_000 }) {
+    ensureDirs();
+    const at = Date.now();
+    for (const contentHash of uploadEntries(resourceId)) {
+      const content = manifest(contentHash);
+      if (!content) throw new Error(`entity content ${contentHash} disappeared from the local cache`);
+      const upload = readUpload(resourceId, contentHash, content);
+      if (!upload || upload.manifestUploaded) continue;
+      if (upload.leaseUntil && Date.parse(upload.leaseUntil) > at) continue;
+      const next = { ...upload, leaseUntil: new Date(at + Math.max(1_000, Number(leaseMs) || 30_000)).toISOString(), updatedAt: now() };
+      if (next.nextPart < content.chunks.length) {
+        const index = next.nextPart;
+        const chunk = content.chunks[index];
+        const bytes = fs.readFileSync(chunkFile(chunk.sha256));
+        if (bytes.length !== chunk.bytes || digest(bytes) !== chunk.sha256) {
+          throw new Error(`local entity content chunk ${chunk.sha256} is corrupt`);
+        }
+        saveUpload(next);
+        return {
+          resourceId,
+          contentHash,
+          kind: 'chunk',
+          partIndex: index,
+          partCount: content.chunks.length,
+          sha256: chunk.sha256,
+          bytes: bytes.length,
+          dataBase64: bytes.toString('base64'),
+        };
+      }
+      const bytes = Buffer.from(JSON.stringify(content), 'utf8');
+      saveUpload(next);
+      return {
+        resourceId,
+        contentHash,
+        kind: 'manifest',
+        partIndex: content.chunks.length,
+        partCount: content.chunks.length,
+        sha256: digest(bytes),
+        bytes: bytes.length,
+        dataBase64: bytes.toString('base64'),
+      };
+    }
+    return null;
+  }
+
+  function acknowledge({ resourceId, contentHash, kind, partIndex }) {
+    const content = manifest(contentHash);
+    if (!content) throw new Error(`entity content ${contentHash} is unavailable`);
+    const current = readUpload(resourceId, contentHash, content);
+    if (!current) throw new Error(`entity content ${contentHash} was never queued`);
+    if (current.manifestUploaded) return { duplicate: true, uploaded: true };
+    if (kind === 'chunk') {
+      if (partIndex !== current.nextPart) {
+        throw new Error(`entity content chunk acknowledgement is out of order for ${contentHash}`);
+      }
+      current.nextPart += 1;
+    } else if (kind === 'manifest') {
+      if (current.nextPart !== content.chunks.length) {
+        throw new Error(`entity content manifest acknowledged before all chunks for ${contentHash}`);
+      }
+      current.manifestUploaded = true;
+    } else {
+      throw new Error('entity content acknowledgement has an unknown kind');
+    }
+    current.leaseUntil = null;
+    current.updatedAt = now();
+    saveUpload(current);
+    return { duplicate: false, uploaded: current.manifestUploaded };
+  }
+
+  function fail({ resourceId, contentHash, message }) {
+    const content = manifest(contentHash);
+    if (!content) throw new Error(`entity content ${contentHash} is unavailable`);
+    const current = readUpload(resourceId, contentHash, content);
+    if (!current || current.manifestUploaded) return { duplicate: Boolean(current?.manifestUploaded) };
+    current.leaseUntil = null;
+    current.lastError = String(message || 'entity content upload failed').slice(0, 4000);
+    current.updatedAt = now();
+    saveUpload(current);
+    return { duplicate: false };
+  }
+
+  function uploaded({ resourceId, contentHash }) {
+    const content = manifest(contentHash);
+    if (!content) return false;
+    return Boolean(readUpload(resourceId, contentHash, content)?.manifestUploaded);
+  }
+
+  function has({ contentHash }) {
+    return manifest(contentHash) !== null;
+  }
+
+  return {
+    captureFile,
+    ensureUpload,
+    queueRecords,
+    claim,
+    acknowledge,
+    fail,
+    uploaded,
+    has,
+  };
+}
+
+module.exports = { CHUNK_BYTES, CONTENT_CONTRACT, createEntityContent };