@mh-gg/git 0.1.1-alpha.20260626T104441232Z

package/src/cas/storage.cjs

@@ -0,0 +1,389 @@
+const fs = require("node:fs");
+const path = require("node:path");
+const { canonicalJson } = require("@mh-gg/event/canonicalJson");
+const { DEFAULT_CHUNKER, chunkBuffer, normalizeChunker } = require("./chunker.cjs");
+const {
+  base64url,
+  chunkAad,
+  deriveEpochSubkeys,
+  fromBase64url,
+  hmacSha256Hex,
+  manifestAad,
+  parseSecret,
+  randomBytes,
+  sha256Hex,
+  xChaCha20Poly1305Open,
+  xChaCha20Poly1305Seal
+} = require("./crypto.cjs");
+const { createLogicalManifestIdentity, createPointer, parsePointer } = require("./manifest.cjs");
+const { dataRootFor, gitDirFromEnv, writeJsonFile } = require("./localMirror.cjs");
+const { originSlug, parseMatterhornOrigin } = require("./originUrl.cjs");
+
+const DEFAULT_PROFILE = "dedup-same-epoch/encrypted-manifest/no-padding";
+
+function ensurePrivateDir(dir) {
+  fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+  try { fs.chmodSync(dir, 0o700); } catch (matterhornIgnoredError) {
+    globalThis.__matterhornIgnoredError?.(matterhornIgnoredError, "packages/matterhorn-git/src/cas/storage.cjs");
+  }
+}
+
+function safeId(value) {
+  return String(value || "").replace(/[^A-Za-z0-9_.-]+/g, "_").slice(0, 240) || "object";
+}
+
+function casRootFor(originInput, options = {}) {
+  const origin = parseMatterhornOrigin(originInput);
+  const gitDir = options.gitDir || gitDirFromEnv(options.cwd);
+  return path.join(dataRootFor(gitDir), "cas", originSlug(origin));
+}
+
+function keyRootFor(originInput, options = {}) {
+  const origin = parseMatterhornOrigin(originInput);
+  const gitDir = options.gitDir || gitDirFromEnv(options.cwd);
+  return path.join(dataRootFor(gitDir), "keys", originSlug(origin));
+}
+
+function indexFileFor(originInput, options = {}) {
+  return path.join(casRootFor(originInput, options), "index.json");
+}
+
+function objectPathFor(originInput, objectId, options = {}) {
+  const root = casRootFor(originInput, options);
+  const id = safeId(objectId);
+  return path.join(root, "objects", id.slice(0, 12), `${id}.json`);
+}
+
+function manifestPathFor(originInput, objectId, options = {}) {
+  const root = casRootFor(originInput, options);
+  const id = safeId(objectId);
+  return path.join(root, "manifests", id.slice(0, 12), `${id}.json`);
+}
+
+function epochRootFileFor(originInput, epoch = 1, options = {}) {
+  return path.join(keyRootFor(originInput, options), `epoch-${Number(epoch) || 1}.key`);
+}
+
+function readJson(file, fallback) {
+  try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return fallback; }
+}
+
+function readCasIndex(originInput, options = {}) {
+  return readJson(indexFileFor(originInput, options), {
+    version: 1,
+    chunks: {},
+    manifests: {},
+    manifestObjects: {},
+    objects: {}
+  });
+}
+
+function writeCasIndex(originInput, index, options = {}) {
+  writeJsonFile(indexFileFor(originInput, options), JSON.parse(canonicalJson({ version: 1, ...index })));
+}
+
+function envEpochRoot(epoch) {
+  return process.env[`MATTERHORN_GIT_EPOCH_ROOT_${Number(epoch) || 1}`] || process.env.MATTERHORN_GIT_EPOCH_ROOT;
+}
+
+function loadOrCreateEpochRoot(originInput, epoch = 1, options = {}) {
+  if (options.epochRoot) return parseSecret(options.epochRoot, "epochRoot");
+  const fromEnv = envEpochRoot(epoch);
+  if (fromEnv) return parseSecret(fromEnv, "MATTERHORN_GIT_EPOCH_ROOT");
+  const file = epochRootFileFor(originInput, epoch, options);
+  try {
+    return parseSecret(fs.readFileSync(file, "utf8"), file);
+  } catch (error) {
+    if (!error || (error.code !== "ENOENT" && error.code !== "ENOTDIR")) throw error;
+  }
+  if (options.create === false) throw new Error(`Missing Matterhorn Git epoch key for epoch ${epoch}; import it with matterhorn-git key import or set MATTERHORN_GIT_EPOCH_ROOT`);
+  ensurePrivateDir(path.dirname(file));
+  const root = randomBytes(32);
+  fs.writeFileSync(file, `${base64url(root)}\n`, { mode: 0o600 });
+  return root;
+}
+
+function importEpochRoot(originInput, epoch, secret, options = {}) {
+  const root = parseSecret(secret, "epoch root");
+  const file = epochRootFileFor(originInput, epoch, options);
+  ensurePrivateDir(path.dirname(file));
+  fs.writeFileSync(file, `${base64url(root)}\n`, { mode: 0o600 });
+  return { file, epoch: Number(epoch) || 1, secret: base64url(root) };
+}
+
+function exportEpochRoot(originInput, epoch = 1, options = {}) {
+  const root = loadOrCreateEpochRoot(originInput, epoch, { ...options, create: false });
+  return base64url(root);
+}
+
+function chunkPlainIdFor(bytes, subkeys) {
+  return `hmac-sha256:${hmacSha256Hex(subkeys.chunkId, Buffer.from("matterhorn.git.chunk-id.v1\0"), bytes)}`;
+}
+
+function objectIdFor(kind, header, nonce, ciphertext, tag) {
+  return `sha256:${sha256Hex(Buffer.from(canonicalJson({ kind, ...header }), "utf8"), nonce, ciphertext, tag)}`;
+}
+
+function fileExists(file) {
+  let fd;
+  try {
+    fd = fs.openSync(file, "r");
+    return true;
+  } catch {
+    return false;
+  } finally {
+    if (fd !== undefined) {
+      try { fs.closeSync(fd); } catch (matterhornIgnoredError) {
+        globalThis.__matterhornIgnoredError?.(matterhornIgnoredError, "packages/matterhorn-git/src/cas/storage.cjs");
+      }
+    }
+  }
+}
+
+function writePrivateJson(file, value) {
+  ensurePrivateDir(path.dirname(file));
+  fs.writeFileSync(file, `${canonicalJson(value)}\n`, { mode: 0o600 });
+}
+
+function createChunkEnvelope({ origin, repoId, epoch, chunker, chunkPlainId, bytes, subkeys }) {
+  const aad = chunkAad({ repoId, epoch, chunkPlainId, clearLen: bytes.length, chunker });
+  const sealed = xChaCha20Poly1305Seal({ key: subkeys.chunkEnc, plaintext: bytes, aad });
+  const header = {
+    version: 1,
+    alg: "rk-git-chunks-v1",
+    aead: sealed.alg,
+    repoId,
+    roomName: origin.room,
+    keyEpoch: Number(epoch),
+    chunkPlainId,
+    clearLen: bytes.length,
+    chunker
+  };
+  const objectId = objectIdFor("matterhorn.git.chunk-object.v1", header, sealed.nonce, sealed.ciphertext, sealed.tag);
+  return {
+    kind: "matterhorn.git.chunk-object.v1",
+    ...header,
+    objectId,
+    storedLen: sealed.ciphertext.length + sealed.tag.length,
+    nonce: base64url(sealed.nonce),
+    ciphertext: base64url(sealed.ciphertext),
+    tag: base64url(sealed.tag)
+  };
+}
+
+function decryptChunkEnvelope(envelope, originInput, options = {}) {
+  if (envelope.kind !== "matterhorn.git.chunk-object.v1") throw new Error("Not a Matterhorn Git chunk object");
+  const origin = parseMatterhornOrigin(originInput, { repo: envelope.repoId, room: envelope.roomName });
+  const epoch = Number(envelope.keyEpoch || envelope.epoch || 1);
+  const epochRoot = loadOrCreateEpochRoot(origin, epoch, { ...options, create: false });
+  const subkeys = deriveEpochSubkeys(epochRoot, { repoId: envelope.repoId || origin.repo, epoch });
+  const nonce = fromBase64url(envelope.nonce, "chunk nonce");
+  const ciphertext = fromBase64url(envelope.ciphertext, "chunk ciphertext");
+  const tag = fromBase64url(envelope.tag, "chunk tag");
+  const aad = chunkAad({ repoId: envelope.repoId || origin.repo, epoch, chunkPlainId: envelope.chunkPlainId, clearLen: envelope.clearLen, chunker: envelope.chunker });
+  const clear = xChaCha20Poly1305Open({ key: subkeys.chunkEnc, nonce, ciphertext, tag, aad });
+  const expectedId = objectIdFor("matterhorn.git.chunk-object.v1", {
+    version: envelope.version,
+    alg: envelope.alg,
+    aead: envelope.aead,
+    repoId: envelope.repoId,
+    roomName: envelope.roomName,
+    keyEpoch: envelope.keyEpoch,
+    chunkPlainId: envelope.chunkPlainId,
+    clearLen: envelope.clearLen,
+    chunker: envelope.chunker
+  }, nonce, ciphertext, tag);
+  if (envelope.objectId !== expectedId) throw new Error("Chunk object hash mismatch");
+  return clear;
+}
+
+function encryptManifest({ origin, repoId, epoch, manifestId, manifestPlaintext, subkeys }) {
+  const plaintext = Buffer.from(canonicalJson(manifestPlaintext), "utf8");
+  const aad = manifestAad({ repoId, epoch, manifestId });
+  const sealed = xChaCha20Poly1305Seal({ key: subkeys.manifestEnc, plaintext, aad });
+  const header = {
+    version: 1,
+    alg: "rk-git-chunks-v1",
+    aead: sealed.alg,
+    repoId,
+    roomName: origin.room,
+    keyEpoch: Number(epoch),
+    manifestId
+  };
+  const objectId = objectIdFor("matterhorn.git.encrypted-manifest.v1", header, sealed.nonce, sealed.ciphertext, sealed.tag);
+  return {
+    kind: "matterhorn.git.encrypted-manifest.v1",
+    ...header,
+    objectId,
+    clearLen: plaintext.length,
+    storedLen: sealed.ciphertext.length + sealed.tag.length,
+    nonce: base64url(sealed.nonce),
+    ciphertext: base64url(sealed.ciphertext),
+    tag: base64url(sealed.tag)
+  };
+}
+
+function decryptManifestEnvelope(envelope, originInput, options = {}) {
+  if (envelope.kind !== "matterhorn.git.encrypted-manifest.v1") throw new Error("Not a Matterhorn Git encrypted manifest");
+  const origin = parseMatterhornOrigin(originInput, { repo: envelope.repoId, room: envelope.roomName });
+  const epoch = Number(envelope.keyEpoch || envelope.epoch || 1);
+  const epochRoot = loadOrCreateEpochRoot(origin, epoch, { ...options, create: false });
+  const subkeys = deriveEpochSubkeys(epochRoot, { repoId: envelope.repoId || origin.repo, epoch });
+  const nonce = fromBase64url(envelope.nonce, "manifest nonce");
+  const ciphertext = fromBase64url(envelope.ciphertext, "manifest ciphertext");
+  const tag = fromBase64url(envelope.tag, "manifest tag");
+  const aad = manifestAad({ repoId: envelope.repoId || origin.repo, epoch, manifestId: envelope.manifestId });
+  const plaintext = xChaCha20Poly1305Open({ key: subkeys.manifestEnc, nonce, ciphertext, tag, aad });
+  const expectedId = objectIdFor("matterhorn.git.encrypted-manifest.v1", {
+    version: envelope.version,
+    alg: envelope.alg,
+    aead: envelope.aead,
+    repoId: envelope.repoId,
+    roomName: envelope.roomName,
+    keyEpoch: envelope.keyEpoch,
+    manifestId: envelope.manifestId
+  }, nonce, ciphertext, tag);
+  if (envelope.objectId !== expectedId) throw new Error("Manifest object hash mismatch");
+  return JSON.parse(plaintext.toString("utf8"));
+}
+
+function upsertIndexForChunk(index, chunkPlainId, object) {
+  const current = index.chunks[chunkPlainId] || { chunkPlainId, objects: [] };
+  const objects = current.objects.filter((item) => item.objectId !== object.objectId);
+  objects.push(object);
+  index.chunks[chunkPlainId] = { chunkPlainId, objects };
+}
+
+function findReusableChunk(index, originInput, epoch, chunkPlainId, options = {}) {
+  const entry = index.chunks?.[chunkPlainId];
+  if (!entry) return undefined;
+  return (entry.objects || []).find((object) => Number(object.epoch) === Number(epoch) && object.objectId && fileExists(objectPathFor(originInput, object.objectId, options)));
+}
+
+function encryptBufferToCas(input, originInput, options = {}) {
+  const origin = parseMatterhornOrigin(originInput);
+  const repoId = origin.repo;
+  const epoch = Number(options.epoch || 1);
+  const fileMode = options.fileMode || "100644";
+  const profile = options.profile || DEFAULT_PROFILE;
+  const chunker = normalizeChunker(options.chunker || DEFAULT_CHUNKER);
+  const buffer = Buffer.isBuffer(input) ? input : Buffer.from(input || "");
+  const epochRoot = loadOrCreateEpochRoot(origin, epoch, options);
+  const subkeys = deriveEpochSubkeys(epochRoot, { repoId, epoch });
+  const index = readCasIndex(origin, options);
+  const manifestChunks = [];
+  const createdObjects = [];
+  const allChunks = chunkBuffer(buffer, chunker);
+
+  for (const chunk of allChunks) {
+    const chunkPlainId = chunkPlainIdFor(chunk.bytes, subkeys);
+    const reusable = options.dedup === "none" ? undefined : findReusableChunk(index, origin, epoch, chunkPlainId, options);
+    let objectMeta;
+    if (reusable) {
+      objectMeta = { ...reusable, reused: true };
+    } else {
+      const envelope = createChunkEnvelope({ origin, repoId, epoch, chunker, chunkPlainId, bytes: chunk.bytes, subkeys });
+      writePrivateJson(objectPathFor(origin, envelope.objectId, options), envelope);
+      objectMeta = { objectId: envelope.objectId, epoch, clearLen: envelope.clearLen, storedLen: envelope.storedLen, createdAt: Date.now() };
+      upsertIndexForChunk(index, chunkPlainId, objectMeta);
+      index.objects[envelope.objectId] = { kind: envelope.kind, epoch, chunkPlainId, clearLen: envelope.clearLen, storedLen: envelope.storedLen, path: objectPathFor(origin, envelope.objectId, options), createdAt: objectMeta.createdAt };
+      createdObjects.push(envelope.objectId);
+    }
+    manifestChunks.push({
+      index: chunk.index,
+      chunkPlainId,
+      clearLen: chunk.clearLen,
+      objectCandidates: [objectMeta.objectId]
+    });
+  }
+
+  const manifestId = createLogicalManifestIdentity({ repoId, epoch, fileMode, clearSize: buffer.length, chunker, chunks: manifestChunks }, subkeys.manifestId);
+  const manifestPlaintext = {
+    kind: "matterhorn.git.file-manifest.v1",
+    version: 1,
+    repoId,
+    roomName: origin.room,
+    keyEpoch: epoch,
+    file: { mode: fileMode, clearSize: buffer.length },
+    profile,
+    alg: {
+      chunker,
+      chunkId: "hmac-sha256",
+      objectHash: "sha256",
+      aead: "xchacha20-poly1305-ietf",
+      compression: "none"
+    },
+    chunks: manifestChunks,
+    logicalHash: `sha256:${sha256Hex(buffer)}`
+  };
+
+  let manifestObjectId;
+  const knownManifest = index.manifests[manifestId];
+  if (knownManifest && fileExists(manifestPathFor(origin, knownManifest.objectId, options))) {
+    manifestObjectId = knownManifest.objectId;
+  } else {
+    const manifestEnvelope = encryptManifest({ origin, repoId, epoch, manifestId, manifestPlaintext, subkeys });
+    writePrivateJson(manifestPathFor(origin, manifestEnvelope.objectId, options), manifestEnvelope);
+    manifestObjectId = manifestEnvelope.objectId;
+    index.manifestObjects[manifestObjectId] = { kind: manifestEnvelope.kind, epoch, manifestId, clearLen: manifestEnvelope.clearLen, storedLen: manifestEnvelope.storedLen, path: manifestPathFor(origin, manifestObjectId, options), createdAt: Date.now() };
+  }
+
+  index.manifests[manifestId] = {
+    manifestId,
+    objectId: manifestObjectId,
+    repoId,
+    epoch,
+    clearSize: buffer.length,
+    chunkCount: manifestChunks.length,
+    pointerProfile: profile,
+    updatedAt: Date.now(),
+    chunks: manifestChunks.map((chunk) => ({ index: chunk.index, chunkPlainId: chunk.chunkPlainId, clearLen: chunk.clearLen, objectCandidates: chunk.objectCandidates }))
+  };
+  writeCasIndex(origin, index, options);
+
+  const pointer = createPointer({ repoId, epoch, manifestId, clearSize: buffer.length, chunkCount: manifestChunks.length, profile });
+  return {
+    origin,
+    repoId,
+    epoch,
+    pointer,
+    pointerFields: parsePointer(pointer),
+    manifestId,
+    manifestObjectId,
+    clearSize: buffer.length,
+    chunkCount: manifestChunks.length,
+    chunks: manifestChunks,
+    createdObjects,
+    profile
+  };
+}
+
+module.exports = {
+  DEFAULT_CHUNKER,
+  DEFAULT_PROFILE,
+  ensurePrivateDir,
+  safeId,
+  casRootFor,
+  keyRootFor,
+  indexFileFor,
+  objectPathFor,
+  manifestPathFor,
+  epochRootFileFor,
+  readJson,
+  readCasIndex,
+  writeCasIndex,
+  envEpochRoot,
+  loadOrCreateEpochRoot,
+  importEpochRoot,
+  exportEpochRoot,
+  chunkPlainIdFor,
+  fileExists,
+  writePrivateJson,
+  createChunkEnvelope,
+  decryptChunkEnvelope,
+  encryptManifest,
+  decryptManifestEnvelope,
+  upsertIndexForChunk,
+  findReusableChunk,
+  encryptBufferToCas
+};

package/src/cas.cjs

@@ -0,0 +1,26 @@
+const storage = require("./cas/storage.cjs");
+const pointer = require("./cas/pointer.cjs");
+
+module.exports = {
+  DEFAULT_CHUNKER: storage.DEFAULT_CHUNKER,
+  DEFAULT_PROFILE: storage.DEFAULT_PROFILE,
+  casRootFor: storage.casRootFor,
+  chunkPlainIdFor: storage.chunkPlainIdFor,
+  decryptChunkEnvelope: storage.decryptChunkEnvelope,
+  decryptManifestEnvelope: storage.decryptManifestEnvelope,
+  encryptBufferToCas: storage.encryptBufferToCas,
+  epochRootFileFor: storage.epochRootFileFor,
+  exportEpochRoot: storage.exportEpochRoot,
+  importEpochRoot: storage.importEpochRoot,
+  indexFileFor: storage.indexFileFor,
+  isMatterhornPointer: pointer.isMatterhornPointer,
+  loadOrCreateEpochRoot: storage.loadOrCreateEpochRoot,
+  manifestEnvelopeForPointer: pointer.manifestEnvelopeForPointer,
+  manifestPathFor: storage.manifestPathFor,
+  materializeCasObjectOutbox: pointer.materializeCasObjectOutbox,
+  metadataForPointer: pointer.metadataForPointer,
+  objectPathFor: storage.objectPathFor,
+  readCasIndex: storage.readCasIndex,
+  restorePointerFromCas: pointer.restorePointerFromCas,
+  writeCasIndex: storage.writeCasIndex
+};

package/src/chunker.cjs

@@ -0,0 +1,67 @@
+const crypto = require("node:crypto");
+
+const DEFAULT_CHUNKER = Object.freeze({
+  name: "fastcdc-js",
+  version: "v1",
+  min: 4 * 1024,
+  target: 16 * 1024,
+  max: 64 * 1024
+});
+
+let gearTable;
+
+function normalizeChunker(input = {}) {
+  const out = {
+    name: input.name || DEFAULT_CHUNKER.name,
+    version: input.version || DEFAULT_CHUNKER.version,
+    min: Number(input.min || DEFAULT_CHUNKER.min),
+    target: Number(input.target || DEFAULT_CHUNKER.target),
+    max: Number(input.max || DEFAULT_CHUNKER.max)
+  };
+  if (!Number.isInteger(out.min) || out.min < 128) throw new Error("chunker.min must be an integer >= 128");
+  if (!Number.isInteger(out.target) || out.target < out.min) throw new Error("chunker.target must be an integer >= min");
+  if (!Number.isInteger(out.max) || out.max < out.target) throw new Error("chunker.max must be an integer >= target");
+  return out;
+}
+
+function gear() {
+  if (gearTable) return gearTable;
+  gearTable = Array.from({ length: 256 }, (_, i) => {
+    const digest = crypto.createHash("sha256").update(`matterhorn.fastcdc-js.v1.gear.${i}`).digest();
+    return digest.readBigUInt64LE(0);
+  });
+  return gearTable;
+}
+
+function maskForTarget(target) {
+  const bits = Math.max(8, Math.round(Math.log2(target)));
+  return (1n << BigInt(bits)) - 1n;
+}
+
+function chunkBuffer(input, chunkerInput = {}) {
+  const buffer = Buffer.isBuffer(input) ? input : Buffer.from(input || "");
+  const chunker = normalizeChunker(chunkerInput);
+  if (buffer.length === 0) return [];
+  const table = gear();
+  const mask = maskForTarget(chunker.target);
+  const chunks = [];
+  let start = 0;
+  while (start < buffer.length) {
+    let fp = 0n;
+    let end = Math.min(buffer.length, start + chunker.max);
+    for (let pos = start; pos < end; pos += 1) {
+      fp = ((fp << 1n) + table[buffer[pos]]) & 0xffffffffffffffffn;
+      const len = pos - start + 1;
+      if (len >= chunker.min && ((fp & mask) === 0n || len >= chunker.max)) {
+        end = pos + 1;
+        break;
+      }
+    }
+    const bytes = buffer.subarray(start, end);
+    chunks.push({ index: chunks.length, offset: start, clearLen: bytes.length, bytes });
+    start = end;
+  }
+  return chunks;
+}
+
+module.exports = { DEFAULT_CHUNKER, chunkBuffer, normalizeChunker };