git-fs-s3 0.3.13 → 0.4.0

package/dist/http.js

@@ -1,6 +1,12 @@
 import {
+  packFileName,
+  parsePackDirectory
+} from "./chunk-RMAOPWHJ.js";
+import {
+  listBranches,
+  listLooseRefNames,
   resolveLooseRefFast
-} from "./chunk-3BDDEGAP.js";
+} from "./chunk-Z4656PGE.js";
 import {
   concat,
   decodeAscii,
@@ -9,7 +15,8 @@ import {
   encodeUtf8,
   fromHex,
   isSafeFullRefName,
-  sha1
+  sha1,
+  toHex
 } from "./chunk-YQFNY6PG.js";
 
 // src/http/info-refs.ts
@@ -73,34 +80,24 @@ function rawFs(repo) {
 
 // src/http/info-refs.ts
 async function listAllRefs(repo, defaultBranch = "main") {
-  const [branches, tags, headSymref] = await Promise.all([
-    git.listBranches(repo),
-    git.listTags(repo),
-    // Wrap with Promise.resolve so a mock/stub returning undefined doesn't crash .then()
-    Promise.resolve(git.currentBranch({ ...repo, fullname: true })).then((cb) => cb ?? `refs/heads/${defaultBranch}`).catch(() => `refs/heads/${defaultBranch}`)
-  ]);
-  const [branchRefs, tagRefs] = await Promise.all([
-    Promise.all(
-      branches.map(async (branch) => {
-        try {
-          const oid = await resolveLooseRefFast(repo, `refs/heads/${branch}`);
-          return { name: `refs/heads/${branch}`, oid };
-        } catch {
-          return null;
-        }
-      })
-    ),
-    Promise.all(
-      tags.map(async (tag) => {
-        try {
-          const oid = await resolveLooseRefFast(repo, `refs/tags/${tag}`);
-          return { name: `refs/tags/${tag}`, oid };
-        } catch {
-          return null;
-        }
-      })
+  const [branches, tagNames] = await Promise.all([
+    listBranches(repo),
+    listLooseRefNames(repo, "refs/tags").then(
+      (names) => names ?? git.listTags(repo)
     )
   ]);
+  const tagRefs = await Promise.all(
+    tagNames.map(async (tag) => {
+      const name = `refs/tags/${tag}`;
+      return resolveLooseRefFast(repo, name).then((oid) => ({ name, oid })).catch(() => null);
+    })
+  );
+  const branchRefs = branches.map((b) => ({
+    name: `refs/heads/${b.name}`,
+    oid: b.commit
+  }));
+  const defaultRef = branches.find((b) => b.isDefault);
+  const headSymref = defaultRef ? `refs/heads/${defaultRef.name}` : await Promise.resolve(git.currentBranch({ ...repo, fullname: true })).then((cb) => cb ?? `refs/heads/${defaultBranch}`).catch(() => `refs/heads/${defaultBranch}`);
   const headOid = branchRefs.find((r) => r?.name === headSymref)?.oid ?? await git.resolveRef({ ...repo, ref: "HEAD" }).catch(() => null);
   const refs = [];
   if (headOid) refs.push({ name: "HEAD", oid: headOid });
@@ -153,7 +150,8 @@ async function handleInfoRefs(repo, options, hooks) {
 import git2 from "isomorphic-git";
 var delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 var MISSING_OBJECT_RETRY_DELAY_MS = 200;
-async function collectReachableOids(repo, startOids, hooks, haveOids) {
+async function collectReachableOids(repo, startOids, hooks, haveOids, options) {
+  const verifyBlobs = options?.verifyBlobs ?? true;
   const have = haveOids instanceof Set ? haveOids : new Set(haveOids ?? []);
   const seen = /* @__PURE__ */ new Set();
   let complete = true;
@@ -161,23 +159,26 @@ async function collectReachableOids(repo, startOids, hooks, haveOids) {
   async function readAndVisitChildren(oid) {
     const obj = await git2.readObject({ ...repo, oid });
     seen.add(oid);
-    let children = [];
+    const children = [];
     if (obj.type === "commit") {
-      const { commit } = await git2.readCommit({ ...repo, oid });
-      children = [commit.tree, ...commit.parent];
+      const commit = obj.object;
+      children.push([commit.tree, "tree"]);
+      for (const parent of commit.parent) children.push([parent, "commit"]);
     } else if (obj.type === "tree") {
-      const { tree } = await git2.readTree({ ...repo, oid });
-      children = tree.map((e) => e.oid);
+      const tree = obj.object;
+      for (const entry of tree) {
+        if (entry.type === "commit") continue;
+        children.push([entry.oid, entry.type]);
+      }
     } else if (obj.type === "tag") {
-      const { tag } = await git2.readTag({ ...repo, oid });
-      children = [tag.object];
+      children.push([obj.object.object, void 0]);
     }
-    await Promise.all(children.map(visit));
+    await Promise.all(children.map(([child, type]) => visit(child, type)));
   }
-  function visit(oid) {
+  function visit(oid, type) {
     const existing = promises.get(oid);
     if (existing) return existing;
-    if (have.has(oid)) {
+    if (have.has(oid) || type === "blob" && !verifyBlobs) {
       seen.add(oid);
       const p2 = Promise.resolve();
       promises.set(oid, p2);
@@ -199,7 +200,7 @@ async function collectReachableOids(repo, startOids, hooks, haveOids) {
     promises.set(oid, p);
     return p;
   }
-  await Promise.all(startOids.map(visit));
+  await Promise.all(startOids.map((oid) => visit(oid)));
   return { oids: Array.from(seen), complete };
 }
 
@@ -208,6 +209,167 @@ import git4 from "isomorphic-git";
 
 // src/http/repack.ts
 import git3 from "isomorphic-git";
+
+// src/http/pack-reuse.ts
+var OBJ_OFS_DELTA = 6;
+var OBJ_REF_DELTA = 7;
+var OID_BYTES = 20;
+var PACK_TRAILER_BYTES = 20;
+var IDX_V2 = {
+  magic: 4285812579,
+  version: 2,
+  headerBytes: 8,
+  fanoutEntries: 256,
+  /** Offsets with this bit set index the 64-bit large-offset table. */
+  largeOffsetFlag: 2147483648
+};
+function readPackIndex(idx) {
+  const view = new DataView(idx.buffer, idx.byteOffset, idx.byteLength);
+  const namesStart = IDX_V2.headerBytes + IDX_V2.fanoutEntries * 4;
+  if (idx.byteLength < namesStart || view.getUint32(0) !== IDX_V2.magic || view.getUint32(4) !== IDX_V2.version) {
+    return null;
+  }
+  const count = view.getUint32(namesStart - 4);
+  const crcStart = namesStart + count * OID_BYTES;
+  const offsetsStart = crcStart + count * 4;
+  if (idx.byteLength < offsetsStart + count * 4) return null;
+  const entries = [];
+  for (let i = 0; i < count; i++) {
+    const offset = view.getUint32(offsetsStart + i * 4);
+    if (offset & IDX_V2.largeOffsetFlag) return null;
+    const at = namesStart + i * OID_BYTES;
+    entries.push({ oid: toHex(idx.subarray(at, at + OID_BYTES)), offset });
+  }
+  return entries;
+}
+function indexPack(idx, pack) {
+  const located = readPackIndex(idx);
+  if (!located) return null;
+  located.sort((a, b) => a.offset - b.offset);
+  const entries = /* @__PURE__ */ new Map();
+  const oidAtOffset = /* @__PURE__ */ new Map();
+  const packEnd = pack.byteLength - PACK_TRAILER_BYTES;
+  for (let i = 0; i < located.length; i++) {
+    const { oid, offset } = located[i];
+    const end = located[i + 1]?.offset ?? packEnd;
+    entries.set(oid, { pack, start: offset, end });
+    oidAtOffset.set(offset, oid);
+  }
+  return { entries, oidAtOffset };
+}
+function readEntryHeader(bytes, start) {
+  let byte = bytes[start];
+  const type = byte >> 4 & 7;
+  let size = byte & 15;
+  let shift = 4;
+  let at = start + 1;
+  while (byte & 128) {
+    byte = bytes[at++];
+    size += (byte & 127) * 2 ** shift;
+    shift += 7;
+  }
+  return { type, size, length: at - start };
+}
+function encodeEntryHeader(type, size) {
+  const bytes = [];
+  let rest = Math.floor(size / 16);
+  bytes.push((rest > 0 ? 128 : 0) | type << 4 | size % 16);
+  while (rest > 0) {
+    const low = rest % 128;
+    rest = Math.floor(rest / 128);
+    bytes.push((rest > 0 ? 128 : 0) | low);
+  }
+  return new Uint8Array(bytes);
+}
+function readOfsDeltaOffset(bytes, start) {
+  let at = start;
+  let byte = bytes[at++];
+  let distance = byte & 127;
+  while (byte & 128) {
+    byte = bytes[at++];
+    distance = (distance + 1) * 128 + (byte & 127);
+  }
+  return { distance, length: at - start };
+}
+function reusableEntry(entry, indexed) {
+  const { pack, start, end } = entry;
+  const header = readEntryHeader(pack, start);
+  const bodyStart = start + header.length;
+  if (header.type === OBJ_REF_DELTA) {
+    return {
+      bytes: pack.subarray(start, end),
+      baseOid: toHex(pack.subarray(bodyStart, bodyStart + OID_BYTES))
+    };
+  }
+  if (header.type !== OBJ_OFS_DELTA) {
+    return { bytes: pack.subarray(start, end), baseOid: null };
+  }
+  const { distance, length } = readOfsDeltaOffset(pack, bodyStart);
+  const baseOid = indexed.oidAtOffset.get(start - distance);
+  if (baseOid === void 0) return null;
+  return {
+    bytes: concat(
+      encodeEntryHeader(OBJ_REF_DELTA, header.size),
+      fromHex(baseOid),
+      pack.subarray(bodyStart + length, end)
+    ),
+    baseOid
+  };
+}
+async function buildPackFromExistingPacks(repo, oids) {
+  const fsp = rawFs(repo);
+  const packDir = `${repo.gitdir}/objects/pack`;
+  const packs = parsePackDirectory(
+    await fsp.readdir(packDir).catch(() => [])
+  ).filter((file) => file.kind === "pack");
+  const indexes = [];
+  for (const { base } of packs) {
+    const [pack, idx] = await Promise.all([
+      fsp.readFile(`${packDir}/${packFileName(base, "pack")}`),
+      fsp.readFile(`${packDir}/${packFileName(base, "idx")}`)
+    ]).catch(() => [null, null]);
+    if (!(pack instanceof Uint8Array) || !(idx instanceof Uint8Array)) {
+      return null;
+    }
+    const indexed = indexPack(idx, pack);
+    if (!indexed) return null;
+    indexes.push(indexed);
+  }
+  const locate = (oid) => {
+    for (const indexed of indexes) {
+      const entry = indexed.entries.get(oid);
+      if (entry) return reusableEntry(entry, indexed);
+    }
+    return null;
+  };
+  const ordered = [];
+  const done = /* @__PURE__ */ new Set();
+  const pending = /* @__PURE__ */ new Set();
+  const visit = (oid) => {
+    if (done.has(oid)) return true;
+    if (pending.has(oid)) return false;
+    const entry = locate(oid);
+    if (!entry) return false;
+    pending.add(oid);
+    if (entry.baseOid !== null && !visit(entry.baseOid)) return false;
+    pending.delete(oid);
+    done.add(oid);
+    ordered.push(entry.bytes);
+    return true;
+  };
+  for (const oid of oids) {
+    if (!visit(oid)) return null;
+  }
+  const header = new Uint8Array(12);
+  header.set(encodeUtf8("PACK"), 0);
+  const view = new DataView(header.buffer);
+  view.setUint32(4, 2);
+  view.setUint32(8, ordered.length);
+  const body = concat(header, ...ordered);
+  return concat(body, fromHex(await sha1(body)));
+}
+
+// src/http/repack.ts
 var PACK_OBJECT_TYPE_BITS = {
   commit: 16,
   tree: 32,
@@ -291,13 +453,13 @@ async function buildVerifiedPack(oids, objects) {
   return concat(packBody, trailer);
 }
 var REPACK_PACK_COUNT_THRESHOLD = 4;
+async function readPackDirectory(repo) {
+  const entries = await rawFs(repo).readdir(`${repo.gitdir}/objects/pack`).catch(() => []);
+  return parsePackDirectory(entries);
+}
 async function countPacks(repo) {
-  try {
-    const entries = await rawFs(repo).readdir(`${repo.gitdir}/objects/pack`);
-    return entries.filter((f) => f.endsWith(".pack")).length;
-  } catch {
-    return 0;
-  }
+  const files = await readPackDirectory(repo);
+  return files.filter((file) => file.kind === "pack").length;
 }
 async function repackRepository(repo, options, hooks) {
   const threshold = options?.threshold ?? REPACK_PACK_COUNT_THRESHOLD;
@@ -320,41 +482,48 @@ async function repackRepository(repo, options, hooks) {
       )
     )).filter((oid) => oid !== null);
     if (tipOids.length === 0) return [];
-    const { oids, complete } = await collectReachableOids(repo, tipOids, hooks);
-    if (!complete || oids.length === 0) return [];
-    const objects = await readAndVerifyObjects(repo, oids);
-    const packBuffer = await buildVerifiedPack(oids, objects);
+    const { oids: reachable, complete } = await collectReachableOids(
+      repo,
+      tipOids,
+      hooks
+    );
+    if (!complete || reachable.length === 0) return [];
     const packDir = `${repo.gitdir}/objects/pack`;
+    const packBuffer = await buildPackFromExistingPacks(repo, reachable) ?? await buildVerifiedPack(
+      reachable,
+      await readAndVerifyObjects(repo, reachable)
+    );
     await fsp.mkdir(packDir, { recursive: true });
     const newBase = `pack-${Date.now()}`;
-    const newPackFile = `${newBase}.pack`;
-    const newIdxFile = `${newBase}.idx`;
+    const newPackFile = packFileName(newBase, "pack");
     await fsp.writeFile(`${packDir}/${newPackFile}`, packBuffer);
     const { oids: indexedOids } = await git3.indexPack({
       ...repo,
       dir: packDir,
       filepath: newPackFile
     });
-    const expected = new Set(oids);
     const indexed = new Set(indexedOids);
-    if (indexed.size !== expected.size || oids.some((oid) => !indexed.has(oid))) {
-      await fsp.unlink(`${packDir}/${newPackFile}`).catch(() => {
-      });
-      await fsp.unlink(`${packDir}/${newIdxFile}`).catch(() => {
-      });
+    if (reachable.some((oid) => !indexed.has(oid))) {
+      await Promise.all(
+        ["pack", "idx"].map(
+          (kind) => fsp.unlink(`${packDir}/${packFileName(newBase, kind)}`).catch(() => {
+          })
+        )
+      );
       throw new Error(
-        "repack: indexed pack's oid set didn't match the verified reachable set \u2014 aborting"
+        "repack: indexed pack is missing reachable objects \u2014 aborting"
       );
     }
-    const allEntries = await fsp.readdir(packDir).catch(() => []);
-    const staleFiles = allEntries.filter(
-      (f) => f !== newPackFile && f !== newIdxFile && (f.endsWith(".pack") || f.endsWith(".idx") || f.endsWith(".keep"))
+    const stale = (await readPackDirectory(repo)).filter(
+      (file) => file.base !== newBase
     );
     await Promise.all(
-      staleFiles.map((f) => fsp.unlink(`${packDir}/${f}`).catch(() => {
-      }))
+      stale.map(
+        (file) => fsp.unlink(`${packDir}/${file.name}`).catch(() => {
+        })
+      )
     );
-    return staleFiles.map((f) => `objects/pack/${f}`);
+    return stale.map((file) => `objects/pack/${file.name}`);
   } catch (err) {
     hooks?.onWarn?.("repack failed (non-fatal)", err);
     return [];
@@ -589,7 +758,7 @@ async function handleUploadPack(repo, body, options, hooks) {
       "readdir objects/pack",
       () => rawFs(repo).readdir(packDirPath).catch(() => [])
     );
-    const packNames = entries.filter((f) => f.endsWith(".pack"));
+    const packNames = parsePackDirectory(entries).filter((file) => file.kind === "pack").map((file) => file.name);
     const packName = packNames[0];
     if (packNames.length === 1 && packName !== void 0) {
       const packData = await runStep(
@@ -611,27 +780,34 @@ async function handleUploadPack(repo, body, options, hooks) {
   if (options?.beforeWalk) {
     await runStep(hooks, "beforeWalk", options.beforeWalk);
   }
-  const { oids: wantOids } = await runStep(
-    hooks,
-    "collectReachableOids(wants)",
-    () => collectReachableOids(repo, wants, hooks)
-  );
-  let oids = wantOids;
+  let haveSet;
   if (haves.length > 0) {
     const { oids: haveOids } = await runStep(
       hooks,
       "collectReachableOids(haves)",
-      () => collectReachableOids(repo, haves, hooks)
+      () => collectReachableOids(repo, haves, hooks, void 0, {
+        verifyBlobs: false
+      })
     );
-    const haveSet = new Set(haveOids);
-    oids = oids.filter((oid) => !haveSet.has(oid));
+    haveSet = new Set(haveOids);
   }
-  const { packfile } = await runStep(
+  const { oids: wantOids } = await runStep(
+    hooks,
+    "collectReachableOids(wants)",
+    () => collectReachableOids(repo, wants, hooks, haveSet, {
+      verifyBlobs: false
+    })
+  );
+  const oids = haveSet ? wantOids.filter((oid) => !haveSet.has(oid)) : wantOids;
+  const reused = await runStep(
     hooks,
-    `packObjects (${oids.length} oids)`,
-    () => git5.packObjects({ ...repo, oids })
+    `reuse pack entries (${oids.length} oids)`,
+    () => buildPackFromExistingPacks(repo, oids)
   );
-  const packBytes = packfile instanceof Uint8Array ? packfile : new Uint8Array(packfile ?? []);
+  const packBytes = reused ?? await runStep(hooks, `packObjects (${oids.length} oids)`, async () => {
+    const { packfile } = await git5.packObjects({ ...repo, oids });
+    return packfile instanceof Uint8Array ? packfile : new Uint8Array(packfile ?? []);
+  });
   return {
     status: 200,
     headers: {