isomorphic-git 1.44.5 → 1.44.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4179,6 +4179,69 @@ function otherVarIntDecode(reader, startWith) {
4179
4179
  return result
4180
4180
  }
4181
4181
 
4182
+ // Memory that `fromPack` may use to keep resolved objects for delta bases.
4183
+ // An object larger than a quarter of this is not kept.
4184
+ const RESOLVE_CACHE_BYTES = 32 * 1024 * 1024;
4185
+
4186
+ // A cache that drops the least recently used object when it holds too many bytes.
4187
+ class ObjectCache {
4188
+ constructor(maxBytes) {
4189
+ this.maxBytes = maxBytes;
4190
+ this.bytes = 0;
4191
+ this.map = new Map();
4192
+ }
4193
+
4194
+ get(key) {
4195
+ const value = this.map.get(key);
4196
+ if (value !== undefined) {
4197
+ this.map.delete(key);
4198
+ this.map.set(key, value);
4199
+ }
4200
+ return value
4201
+ }
4202
+
4203
+ set(key, value) {
4204
+ const size = value.object.byteLength;
4205
+ if (size > this.maxBytes / 4) return
4206
+ this.map.set(key, value);
4207
+ this.bytes += size;
4208
+ for (const [oldKey, old] of this.map) {
4209
+ if (this.bytes <= this.maxBytes) break
4210
+ this.map.delete(oldKey);
4211
+ this.bytes -= old.object.byteLength;
4212
+ }
4213
+ }
4214
+ }
4215
+
4216
+ // Resolve an object from the data that `listpack` already inflated.
4217
+ // Return null when the object needs `readSlice`. This happens when the base of
4218
+ // a delta is not in the cache, or when the delta is corrupt.
4219
+ function resolveInflated(
4220
+ { type, data, reference, offset },
4221
+ cache,
4222
+ offsetByOid
4223
+ ) {
4224
+ // `applyDelta` and `shasum` need a Buffer. This is a view and copies nothing.
4225
+ const object = Buffer.from(data.buffer, data.byteOffset, data.byteLength);
4226
+ if (type !== 'ofs-delta' && type !== 'ref-delta') return { type, object }
4227
+ const baseOffset =
4228
+ type === 'ofs-delta'
4229
+ ? offset - decodeVarInt(new BufferCursor(reference))
4230
+ : offsetByOid.get(reference.toString('hex'));
4231
+ const base = cache.get(baseOffset);
4232
+ if (!base) return null
4233
+ try {
4234
+ return {
4235
+ type: base.type,
4236
+ object: Buffer.from(applyDelta(object, base.object)),
4237
+ }
4238
+ } catch (err) {
4239
+ // A corrupt delta. `readSlice` reports it again. Any other error is a bug.
4240
+ if (err instanceof InternalError || err instanceof RangeError) return null
4241
+ throw err
4242
+ }
4243
+ }
4244
+
4182
4245
  class GitPackIndex {
4183
4246
  constructor(stuff) {
4184
4247
  Object.assign(this, stuff);
@@ -4250,6 +4313,9 @@ class GitPackIndex {
4250
4313
  const offsets = new Map();
4251
4314
  let totalObjectCount = null;
4252
4315
  let lastPercent = null;
4316
+ // Objects that `listpack` inflated, by pack offset. A delta finds its base here.
4317
+ const cache = new ObjectCache(RESOLVE_CACHE_BYTES);
4318
+ const offsetByOid = new Map();
4253
4319
 
4254
4320
  await listpack([pack], async ({ data, type, reference, offset, num }) => {
4255
4321
  if (totalObjectCount === null) totalObjectCount = num;
@@ -4285,6 +4351,20 @@ class GitPackIndex {
4285
4351
  offset,
4286
4352
  };
4287
4353
  }
4354
+
4355
+ // The object is already inflated, so get its oid now. Then the loop below
4356
+ // does not inflate it again. That loop reads all other objects.
4357
+ const o = offsetToObject[offset];
4358
+ if (!o) return
4359
+ const resolved = resolveInflated(
4360
+ { type, data, reference, offset },
4361
+ cache,
4362
+ offsetByOid
4363
+ );
4364
+ if (!resolved) return
4365
+ o.oid = await shasum(GitObject.wrap(resolved));
4366
+ offsetByOid.set(o.oid, offset);
4367
+ cache.set(offset, resolved);
4288
4368
  });
4289
4369
 
4290
4370
  // We need to know the lengths of the slices to compute the CRCs.
@@ -4328,20 +4408,21 @@ class GitPackIndex {
4328
4408
  lastPercent = percent;
4329
4409
 
4330
4410
  const o = offsetToObject[offset];
4331
- if (o.oid) continue
4332
- try {
4333
- p.readDepth = 0;
4334
- p.externalReadDepth = 0;
4335
- const { type, object } = await p.readSlice({ start: offset });
4336
- objectsByDepth[p.readDepth] += 1;
4337
- const oid = await shasum(GitObject.wrap({ type, object }));
4338
- o.oid = oid;
4339
- hashes.push(oid);
4340
- offsets.set(oid, offset);
4341
- crcs[oid] = o.crc;
4342
- } catch (err) {
4343
- continue
4411
+ if (!o.oid) {
4412
+ // The first pass did not resolve this object, so read it.
4413
+ try {
4414
+ p.readDepth = 0;
4415
+ p.externalReadDepth = 0;
4416
+ const { type, object } = await p.readSlice({ start: offset });
4417
+ objectsByDepth[p.readDepth] += 1;
4418
+ o.oid = await shasum(GitObject.wrap({ type, object }));
4419
+ } catch (err) {
4420
+ continue
4421
+ }
4344
4422
  }
4423
+ hashes.push(o.oid);
4424
+ offsets.set(o.oid, offset);
4425
+ crcs[o.oid] = o.crc;
4345
4426
  }
4346
4427
 
4347
4428
  hashes.sort();
package/managers/index.js CHANGED
@@ -4172,6 +4172,69 @@ function otherVarIntDecode(reader, startWith) {
4172
4172
  return result
4173
4173
  }
4174
4174
 
4175
+ // Memory that `fromPack` may use to keep resolved objects for delta bases.
4176
+ // An object larger than a quarter of this is not kept.
4177
+ const RESOLVE_CACHE_BYTES = 32 * 1024 * 1024;
4178
+
4179
+ // A cache that drops the least recently used object when it holds too many bytes.
4180
+ class ObjectCache {
4181
+ constructor(maxBytes) {
4182
+ this.maxBytes = maxBytes;
4183
+ this.bytes = 0;
4184
+ this.map = new Map();
4185
+ }
4186
+
4187
+ get(key) {
4188
+ const value = this.map.get(key);
4189
+ if (value !== undefined) {
4190
+ this.map.delete(key);
4191
+ this.map.set(key, value);
4192
+ }
4193
+ return value
4194
+ }
4195
+
4196
+ set(key, value) {
4197
+ const size = value.object.byteLength;
4198
+ if (size > this.maxBytes / 4) return
4199
+ this.map.set(key, value);
4200
+ this.bytes += size;
4201
+ for (const [oldKey, old] of this.map) {
4202
+ if (this.bytes <= this.maxBytes) break
4203
+ this.map.delete(oldKey);
4204
+ this.bytes -= old.object.byteLength;
4205
+ }
4206
+ }
4207
+ }
4208
+
4209
+ // Resolve an object from the data that `listpack` already inflated.
4210
+ // Return null when the object needs `readSlice`. This happens when the base of
4211
+ // a delta is not in the cache, or when the delta is corrupt.
4212
+ function resolveInflated(
4213
+ { type, data, reference, offset },
4214
+ cache,
4215
+ offsetByOid
4216
+ ) {
4217
+ // `applyDelta` and `shasum` need a Buffer. This is a view and copies nothing.
4218
+ const object = Buffer.from(data.buffer, data.byteOffset, data.byteLength);
4219
+ if (type !== 'ofs-delta' && type !== 'ref-delta') return { type, object }
4220
+ const baseOffset =
4221
+ type === 'ofs-delta'
4222
+ ? offset - decodeVarInt(new BufferCursor(reference))
4223
+ : offsetByOid.get(reference.toString('hex'));
4224
+ const base = cache.get(baseOffset);
4225
+ if (!base) return null
4226
+ try {
4227
+ return {
4228
+ type: base.type,
4229
+ object: Buffer.from(applyDelta(object, base.object)),
4230
+ }
4231
+ } catch (err) {
4232
+ // A corrupt delta. `readSlice` reports it again. Any other error is a bug.
4233
+ if (err instanceof InternalError || err instanceof RangeError) return null
4234
+ throw err
4235
+ }
4236
+ }
4237
+
4175
4238
  class GitPackIndex {
4176
4239
  constructor(stuff) {
4177
4240
  Object.assign(this, stuff);
@@ -4243,6 +4306,9 @@ class GitPackIndex {
4243
4306
  const offsets = new Map();
4244
4307
  let totalObjectCount = null;
4245
4308
  let lastPercent = null;
4309
+ // Objects that `listpack` inflated, by pack offset. A delta finds its base here.
4310
+ const cache = new ObjectCache(RESOLVE_CACHE_BYTES);
4311
+ const offsetByOid = new Map();
4246
4312
 
4247
4313
  await listpack([pack], async ({ data, type, reference, offset, num }) => {
4248
4314
  if (totalObjectCount === null) totalObjectCount = num;
@@ -4278,6 +4344,20 @@ class GitPackIndex {
4278
4344
  offset,
4279
4345
  };
4280
4346
  }
4347
+
4348
+ // The object is already inflated, so get its oid now. Then the loop below
4349
+ // does not inflate it again. That loop reads all other objects.
4350
+ const o = offsetToObject[offset];
4351
+ if (!o) return
4352
+ const resolved = resolveInflated(
4353
+ { type, data, reference, offset },
4354
+ cache,
4355
+ offsetByOid
4356
+ );
4357
+ if (!resolved) return
4358
+ o.oid = await shasum(GitObject.wrap(resolved));
4359
+ offsetByOid.set(o.oid, offset);
4360
+ cache.set(offset, resolved);
4281
4361
  });
4282
4362
 
4283
4363
  // We need to know the lengths of the slices to compute the CRCs.
@@ -4321,20 +4401,21 @@ class GitPackIndex {
4321
4401
  lastPercent = percent;
4322
4402
 
4323
4403
  const o = offsetToObject[offset];
4324
- if (o.oid) continue
4325
- try {
4326
- p.readDepth = 0;
4327
- p.externalReadDepth = 0;
4328
- const { type, object } = await p.readSlice({ start: offset });
4329
- objectsByDepth[p.readDepth] += 1;
4330
- const oid = await shasum(GitObject.wrap({ type, object }));
4331
- o.oid = oid;
4332
- hashes.push(oid);
4333
- offsets.set(oid, offset);
4334
- crcs[oid] = o.crc;
4335
- } catch (err) {
4336
- continue
4404
+ if (!o.oid) {
4405
+ // The first pass did not resolve this object, so read it.
4406
+ try {
4407
+ p.readDepth = 0;
4408
+ p.externalReadDepth = 0;
4409
+ const { type, object } = await p.readSlice({ start: offset });
4410
+ objectsByDepth[p.readDepth] += 1;
4411
+ o.oid = await shasum(GitObject.wrap({ type, object }));
4412
+ } catch (err) {
4413
+ continue
4414
+ }
4337
4415
  }
4416
+ hashes.push(o.oid);
4417
+ offsets.set(o.oid, offset);
4418
+ crcs[o.oid] = o.crc;
4338
4419
  }
4339
4420
 
4340
4421
  hashes.sort();