git-dedup-core 0.1.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,11 +1,11 @@
1
1
  import { execFile, spawn } from 'node:child_process';
2
2
  import { promisify } from 'node:util';
3
3
  import { constants } from 'node:fs';
4
- import { access, mkdir, readdir, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises';
4
+ import { access, mkdir, readdir, readFile, realpath, rename, rm, stat, utimes, writeFile } from 'node:fs/promises';
5
5
  import { homedir, constants as osConstants } from 'node:os';
6
6
  import { basename, delimiter, dirname, join, resolve, sep } from 'node:path';
7
7
  import { createHash, randomUUID } from 'node:crypto';
8
- function cacheFailureReason(error, stage) {
8
+ function addFailureReason(error, stage) {
9
9
  const message = String(error);
10
10
  if (/cannot lock ref|reference already exists|case.insensitive|refname collision/i.test(message))
11
11
  return 'ref collision in the shared pool';
@@ -26,11 +26,14 @@ function delay(ms) {
26
26
  }
27
27
  function keyForRemote(remote) {
28
28
  let host;
29
+ let port = '';
30
+ let user;
29
31
  let pathname;
30
- if (/^[^/@:]+@[^/:]+:.+/.test(remote)) {
31
- const match = remote.match(/^[^/@:]+@([^/:]+):(.+)$/);
32
- host = match[1];
33
- pathname = match[2];
32
+ const scp = remote.match(/^([^/@:]+)@([^/:]+):(.+)$/);
33
+ if (scp) {
34
+ [, user, host, pathname] = scp;
35
+ if (!pathname.startsWith('/'))
36
+ pathname = `/~/${pathname}`;
34
37
  }
35
38
  else {
36
39
  let url;
@@ -42,9 +45,20 @@ function keyForRemote(remote) {
42
45
  }
43
46
  if (!['https:', 'http:', 'ssh:', 'git:'].includes(url.protocol))
44
47
  return undefined;
45
- host = url.hostname + (url.port ? `_${url.port}` : '');
48
+ host = url.hostname;
49
+ // `:` cannot appear in a host name, so ports never collide with hosts.
50
+ port = url.port ? `:${url.port}` : '';
51
+ if (url.protocol === 'ssh:')
52
+ user = url.username;
46
53
  pathname = url.pathname;
47
54
  }
55
+ // Home-relative SSH paths differ per user; `git` is the shared hosting account convention.
56
+ let home = '';
57
+ if (user !== undefined && pathname.startsWith('/~/')) {
58
+ pathname = pathname.slice(2);
59
+ if (user !== 'git')
60
+ home = `~${user}/`;
61
+ }
48
62
  const parts = pathname
49
63
  .replace(/^\/+/, '')
50
64
  .replace(/\.git\/?$/, '')
@@ -55,7 +69,7 @@ function keyForRemote(remote) {
55
69
  return undefined;
56
70
  if (!/^[a-zA-Z0-9._-]+$/.test(host))
57
71
  return undefined;
58
- return `${host.toLowerCase()}/${parts.join('/')}`;
72
+ return `${host.toLowerCase()}${port}/${home}${parts.join('/')}`;
59
73
  }
60
74
  async function directorySize(path) {
61
75
  let size = 0;
@@ -70,11 +84,27 @@ async function directorySize(path) {
70
84
  }
71
85
  const remoteId = (key) => createHash('sha256').update(key).digest('hex');
72
86
  function unsupportedClone(args) {
73
- return args.some((arg) => /^(--depth|--shallow|--filter|--mirror|--bare|--reference|--dissociate|--no-hardlinks|--shared|--separate-git-dir|--bundle-uri|--sparse|--upload-pack|--server-option|--template|--config|--origin|--no-checkout|--single-branch|--no-single-branch|--revision|--jobs|-j|-o|-u|-c)(=|$)/.test(arg));
87
+ return args.find((arg) => /^(--depth|--shallow|--filter|--mirror|--bare|--reference|--dissociate|--no-hardlinks|--shared|--separate-git-dir|--bundle-uri|--sparse|--upload-pack|--server-option|--template|--config|--origin|--no-checkout|--single-branch|--no-single-branch|--revision|--jobs|-j|-o|-u|-c)(=|$)/.test(arg));
88
+ }
89
+ /** Reports a handled command that is forwarded to plain Git, unless Git was asked to be quiet. */
90
+ function fallback(args, reason) {
91
+ if (!args.includes('-q') && !args.includes('--quiet'))
92
+ process.stderr.write(`git-dedup: ${reason}; using plain Git\n`);
93
+ return undefined;
74
94
  }
95
+ /** The last Git `fatal:` or `error:` line of a failure, without captured progress output. */
96
+ function gitFailure(error) {
97
+ const line = String(error)
98
+ .match(/\b(?:fatal|error): [^\r\n]*/g)
99
+ ?.at(-1)
100
+ ?.trim();
101
+ return line ? ` (${line})` : '';
102
+ }
103
+ /** Returns the parsed clone, or the reason git-dedup cannot handle it. */
75
104
  function parseClone(args) {
76
- if (unsupportedClone(args))
77
- return undefined;
105
+ const unsupported = unsupportedClone(args);
106
+ if (unsupported)
107
+ return `clone option ${unsupported.split('=')[0]} is not supported`;
78
108
  const positional = [];
79
109
  let recurse = false;
80
110
  let branch;
@@ -88,7 +118,7 @@ function parseClone(args) {
88
118
  if (arg === '-b' || arg === '--branch') {
89
119
  branch = args[++i];
90
120
  if (!branch)
91
- return undefined;
121
+ return `clone option ${arg} requires a value`;
92
122
  forwarded.push(arg, branch);
93
123
  continue;
94
124
  }
@@ -107,20 +137,42 @@ function parseClone(args) {
107
137
  continue;
108
138
  }
109
139
  if (arg.startsWith('-'))
110
- return undefined;
140
+ return `clone option ${arg.split('=')[0]} is not supported`;
111
141
  positional.push(arg);
112
142
  }
113
143
  if (positional.length < 1 || positional.length > 2)
114
- return undefined;
144
+ return 'clone expects a repository and an optional directory';
145
+ if (positional.includes(''))
146
+ return 'clone does not support an empty argument';
115
147
  return { remote: positional[0], destination: positional[1], recurse, branch, forwarded };
116
148
  }
117
149
  function defaultCloneName(remote) {
150
+ // Like Git, a local `repo/.git` (or `repo//.git`) clones into `repo`.
118
151
  return remote
119
152
  .replace(/\/+$/, '')
153
+ .replace(/\/+\.git$/, '')
120
154
  .split(/[/:]/)
121
155
  .at(-1)
122
156
  .replace(/\.git$/, '');
123
157
  }
158
+ function submoduleAddPath(args) {
159
+ const positional = [];
160
+ for (let i = 1; i < args.length; i++) {
161
+ const arg = args[i];
162
+ if (arg === '--') {
163
+ positional.push(...args.slice(i + 1));
164
+ break;
165
+ }
166
+ // Options of `git submodule add` that take a separate value.
167
+ if (['-b', '--branch', '--reference', '--ref-format', '--name', '--depth'].includes(arg))
168
+ i++;
169
+ else if (!arg.startsWith('-'))
170
+ positional.push(arg);
171
+ }
172
+ if (positional.length === 1)
173
+ return defaultCloneName(positional[0]);
174
+ return positional.length === 2 ? positional[1] : undefined;
175
+ }
124
176
  function resolveSubmoduleRemote(parent, child) {
125
177
  if (!child.startsWith('./') && !child.startsWith('../'))
126
178
  return child;
@@ -163,12 +215,6 @@ function parseGlobal(args, cwd) {
163
215
  i += 2;
164
216
  continue;
165
217
  }
166
- if (arg.startsWith('-C') && arg.length > 2) {
167
- prefix.push(arg);
168
- cwd = resolve(cwd, arg.slice(2));
169
- i++;
170
- continue;
171
- }
172
218
  if (/^(--git-dir|--work-tree|--namespace|--config-env)=/.test(arg) || arg.startsWith('-c')) {
173
219
  prefix.push(arg);
174
220
  i++;
@@ -187,38 +233,91 @@ async function canonicalPath(path) {
187
233
  const parent = dirname(path);
188
234
  return parent === path ? path : join(await canonicalPath(parent), basename(path));
189
235
  }
236
+ // A holder touches its heartbeat file; a waiter that sees the lock unchanged this long treats it as abandoned.
237
+ const LOCK_STALE_MS = 60_000;
190
238
  async function lockDirectory(lock, action) {
191
239
  await mkdir(dirname(lock), { recursive: true });
240
+ let seen = '';
241
+ let seenSince = 0;
192
242
  for (let attempts = 0;; attempts++) {
193
243
  try {
194
244
  await mkdir(lock);
195
- await writeFile(join(lock, 'owner'), `${process.pid}\n`);
196
245
  break;
197
246
  }
198
247
  catch (error) {
199
248
  if (error.code !== 'EEXIST')
200
249
  throw error;
201
- if (attempts > 1200)
202
- throw new Error(`Timed out waiting for git-dedup lock: ${lock}`, { cause: error });
203
- if (attempts === 20)
204
- process.stderr.write('git-dedup: waiting for the object pool lock\n');
205
- const owner = Number((await readFile(join(lock, 'owner'), 'utf8').catch(() => '')).trim());
206
- const age = Date.now() - (await stat(lock).catch(() => ({ mtimeMs: Date.now() }))).mtimeMs;
207
- // A dead owner may be reaped. A live but slow Git process must never lose its lock.
208
- if (age > 5000 && owner && !processExists(owner)) {
209
- await rm(lock, { recursive: true, force: true });
210
- continue;
211
- }
212
- await delay(50);
213
250
  }
251
+ const state = await inspectLock(lock);
252
+ if (attempts === 20)
253
+ process.stderr.write(`git-dedup: waiting for the object pool lock ${lock} (owner pid ${state.owner || 'unknown'})\n`);
254
+ // Monotonic time pauses during sleep, so a holder suspended with the machine is not mistaken for a hung one.
255
+ if (state.key !== seen)
256
+ [seen, seenSince] = [state.key, performance.now()];
257
+ // A dead owner may be reaped. A live but slow Git process must never lose its lock: it keeps its
258
+ // heartbeat fresh. A lock from a version without a heartbeat still waits on a live owner.
259
+ const dead = state.owner > 0 && !processExists(state.owner) && state.age > 5000;
260
+ const silent = (!state.owner || state.heartbeat) && performance.now() - seenSince > LOCK_STALE_MS;
261
+ if (dead || silent)
262
+ await reapLock(lock, state.key);
263
+ await delay(50);
214
264
  }
265
+ const heartbeat = join(lock, 'heartbeat');
266
+ const timer = setInterval(() => {
267
+ const now = new Date();
268
+ void utimes(heartbeat, now, now).catch(() => { });
269
+ }, LOCK_STALE_MS / 6);
270
+ timer.unref();
215
271
  try {
272
+ await writeFile(join(lock, 'owner'), `${process.pid}\n`);
273
+ await writeFile(heartbeat, '');
216
274
  return await action();
217
275
  }
218
276
  finally {
277
+ clearInterval(timer);
219
278
  await rm(lock, { recursive: true, force: true });
220
279
  }
221
280
  }
281
+ async function inspectLock(lock) {
282
+ const [dir, owner, heartbeat] = await Promise.all([
283
+ stat(lock).catch(() => undefined),
284
+ readFile(join(lock, 'owner'), 'utf8').catch(() => ''),
285
+ stat(join(lock, 'heartbeat')).catch(() => undefined),
286
+ ]);
287
+ return {
288
+ key: `${dir?.ino}:${dir?.mtimeMs}:${owner}:${heartbeat?.mtimeMs}`,
289
+ owner: Number(owner.trim()) || 0,
290
+ heartbeat: Boolean(heartbeat),
291
+ age: dir ? Date.now() - dir.mtimeMs : 0,
292
+ };
293
+ }
294
+ /** Removes the lock only if it is still the one the caller judged stale, one reaper at a time. */
295
+ async function reapLock(lock, key) {
296
+ const guard = `${lock}.reap`;
297
+ try {
298
+ await mkdir(guard);
299
+ }
300
+ catch (error) {
301
+ if (error.code !== 'EEXIST')
302
+ throw error;
303
+ // ponytail: a reaper killed mid-reap strands its guard, and two waiters clearing that stranded guard can
304
+ // still race. That needs a crash inside a millisecond window first; give the guard an owner if it ever bites.
305
+ const guardAge = Date.now() - ((await stat(guard).catch(() => undefined))?.mtimeMs ?? Date.now());
306
+ if (guardAge > LOCK_STALE_MS)
307
+ await rm(guard, { recursive: true, force: true });
308
+ return;
309
+ }
310
+ try {
311
+ if ((await inspectLock(lock)).key !== key)
312
+ return;
313
+ const tombstone = `${lock}.${randomUUID()}.reaped`;
314
+ await rename(lock, tombstone).catch(() => { });
315
+ await rm(tombstone, { recursive: true, force: true });
316
+ }
317
+ finally {
318
+ await rm(guard, { recursive: true, force: true });
319
+ }
320
+ }
222
321
  function storeLock(root) {
223
322
  return join(dirname(root), `.${basename(root)}.gitx-lock`);
224
323
  }
@@ -255,19 +354,81 @@ async function withLock(root, action) {
255
354
  function poolPath(root) {
256
355
  return join(root, 'pool.git');
257
356
  }
357
+ // Keep URL credentials out of the store; Git credential helpers supply them at fetch time.
358
+ // HTTP(S) userinfo is a credential. Other schemes keep the user name, which SSH needs.
359
+ function redactRemote(remote, mask = '') {
360
+ return remote.replace(/^([a-z][a-z0-9+.-]*:\/\/)([^/?#]*)@/i, (authority, scheme, userinfo) => {
361
+ const user = /^https?:/i.test(scheme) ? '' : userinfo.split(':')[0];
362
+ if (user === userinfo)
363
+ return authority;
364
+ const kept = user + (mask && (user ? ':' : '') + mask);
365
+ return scheme + (kept && `${kept}@`);
366
+ });
367
+ }
368
+ function registration(remote, key) {
369
+ return JSON.stringify({ key, remote: redactRemote(remote) }) + '\n';
370
+ }
258
371
  async function registerRemote(root, remote, key) {
259
372
  const id = remoteId(key);
260
373
  await mkdir(join(root, 'remotes'), { recursive: true });
261
- await writeFile(join(root, 'remotes', `${id}.json`), JSON.stringify({ key, remote }) + '\n');
374
+ await writeFile(join(root, 'remotes', `${id}.json`), registration(remote, key));
375
+ }
376
+ async function registerConsumer(root, id, gitdir) {
377
+ await mkdir(join(root, 'consumers'), { recursive: true });
378
+ await writeFile(join(root, 'consumers', `${id}.json`), JSON.stringify({ gitdir }) + '\n');
379
+ }
380
+ async function registeredConsumers(root) {
381
+ const consumers = new Map();
382
+ for (const name of await readdir(join(root, 'consumers')).catch(() => [])) {
383
+ const value = JSON.parse(await readFile(join(root, 'consumers', name), 'utf8'));
384
+ const id = name.replace(/\.json$/, '');
385
+ if (!/^[a-f0-9-]{36}$/.test(id) ||
386
+ !value ||
387
+ typeof value !== 'object' ||
388
+ !('gitdir' in value) ||
389
+ typeof value.gitdir !== 'string')
390
+ throw new Error(`Invalid git-dedup consumer registration: ${name}`);
391
+ consumers.set(id, value.gitdir);
392
+ }
393
+ return consumers;
394
+ }
395
+ /** Pseudorefs and in-progress rebase or cherry-pick state can name fetched commits no ref holds. */
396
+ async function stateOids(gitdir) {
397
+ const files = ['FETCH_HEAD', 'ORIG_HEAD', 'MERGE_HEAD', 'CHERRY_PICK_HEAD', 'REVERT_HEAD', 'REBASE_HEAD'].map((name) => join(gitdir, name));
398
+ for (const directory of ['rebase-merge', 'rebase-apply', 'sequencer'])
399
+ for (const entry of await readdir(join(gitdir, directory), { withFileTypes: true }).catch(() => []))
400
+ if (entry.isFile())
401
+ files.push(join(gitdir, directory, entry.name));
402
+ const oids = [];
403
+ for (const file of files)
404
+ oids.push(...((await readFile(file, 'utf8').catch(() => '')).match(/\b[0-9a-f]{40}\b/g) ?? []));
405
+ return oids;
406
+ }
407
+ async function carriesConsumerId(gitdir, id) {
408
+ const current = await readFile(join(gitdir, 'gitx-consumer-id'), 'utf8').catch((error) => {
409
+ if (error.code === 'ENOENT' || error.code === 'ENOTDIR')
410
+ return '';
411
+ throw error;
412
+ });
413
+ return current.trim() === id;
414
+ }
415
+ function checkoutPath(gitdir) {
416
+ return basename(gitdir) === '.git' ? dirname(gitdir) : gitdir;
417
+ }
418
+ function alternateLines(content) {
419
+ return content
420
+ .split('\n')
421
+ .map((line) => line.trim())
422
+ .filter(Boolean);
262
423
  }
263
424
  async function setAlternate(gitdir, pool) {
264
425
  const alternate = join(gitdir, 'objects', 'info', 'alternates');
265
426
  await mkdir(dirname(alternate), { recursive: true });
266
427
  const target = join(pool, 'objects');
267
- const existing = await readFile(alternate, 'utf8').catch(() => '');
268
- if (existing.split('\n').includes(target))
428
+ const lines = alternateLines(await readFile(alternate, 'utf8').catch(() => ''));
429
+ if (lines.includes(target))
269
430
  return;
270
- await writeFile(alternate, existing + target + '\n');
431
+ await writeFile(alternate, [...lines, target].join('\n') + '\n');
271
432
  }
272
433
  async function removeLegacyKeeps(gitdir) {
273
434
  const packDirectory = join(gitdir, 'objects', 'pack');
@@ -279,12 +440,25 @@ async function removeLegacyKeeps(gitdir) {
279
440
  await rm(path);
280
441
  }
281
442
  }
282
- async function consumerId(commonGitdir) {
443
+ /** True when `id` is registered to a different gitdir that still carries it, so `gitdir` is a copy (cp -R). */
444
+ async function ownedElsewhere(root, id, gitdir) {
445
+ const owner = await readFile(join(root, 'consumers', `${id}.json`), 'utf8').then((text) => JSON.parse(text).gitdir, () => undefined);
446
+ return (typeof owner === 'string' &&
447
+ (await canonicalPath(owner)) !== (await canonicalPath(gitdir)) &&
448
+ (await carriesConsumerId(owner, id)));
449
+ }
450
+ async function consumerId(root, commonGitdir) {
283
451
  // Shared across linked worktrees so a tip is pinned only once per object database.
284
452
  const path = join(commonGitdir, 'gitx-consumer-id');
285
453
  const existing = (await readFile(path, 'utf8').catch(() => '')).trim();
286
- if (/^[a-f0-9-]{36}$/.test(existing))
287
- return existing;
454
+ if (/^[a-f0-9-]{36}$/.test(existing)) {
455
+ if (!(await ownedElsewhere(root, existing, commonGitdir)))
456
+ return existing;
457
+ // A copied checkout must not share the original's pins or registration.
458
+ const id = randomUUID();
459
+ await writeFile(path, `${id}\n`);
460
+ return id;
461
+ }
288
462
  const id = randomUUID();
289
463
  try {
290
464
  await writeFile(path, `${id}\n`, { flag: 'wx' });
@@ -297,6 +471,8 @@ async function consumerId(commonGitdir) {
297
471
  }
298
472
  }
299
473
  export function createGitDedup(options = {}) {
474
+ if (process.platform === 'win32')
475
+ throw new Error('git-dedup does not support Windows yet. Run it inside WSL, or use Git directly. See https://github.com/bhouston/git-dedup/issues/74');
300
476
  const cwd = resolve(options.cwd ?? process.cwd());
301
477
  const incomingEnv = { ...process.env, ...options.env };
302
478
  const env = { ...incomingEnv, GITX_ACTIVE: '1', GIT_DEDUP_ACTIVE: '1' };
@@ -346,18 +522,26 @@ export function createGitDedup(options = {}) {
346
522
  await access(candidate, constants.X_OK);
347
523
  return (cachedGit = candidate);
348
524
  }
349
- async function git(args, at = cwd, inherit = false) {
525
+ // 'tee' captures output like the default and also streams stderr (Git progress) to the user.
526
+ async function git(args, at = cwd, inherit = false, input) {
350
527
  const binary = await gitPath();
351
528
  return new Promise((resolveResult, reject) => {
352
- const child = spawn(binary, args, { cwd: at, env, stdio: inherit ? 'inherit' : ['ignore', 'pipe', 'pipe'] });
529
+ const child = spawn(binary, args, {
530
+ cwd: at,
531
+ env,
532
+ stdio: inherit === true ? 'inherit' : [input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'],
533
+ });
534
+ child.stdin?.end(input);
353
535
  let stdout = '';
354
536
  let stderr = '';
355
- if (!inherit) {
537
+ if (inherit !== true) {
356
538
  child.stdout?.on('data', (b) => {
357
539
  stdout += b;
358
540
  });
359
541
  child.stderr?.on('data', (b) => {
360
542
  stderr += b;
543
+ if (inherit === 'tee')
544
+ process.stderr.write(b);
361
545
  });
362
546
  }
363
547
  const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
@@ -365,10 +549,10 @@ export function createGitDedup(options = {}) {
365
549
  child.kill(signal);
366
550
  });
367
551
  const cleanup = () => {
368
- if (inherit)
552
+ if (inherit === true)
369
553
  signals.forEach((signal, i) => process.removeListener(signal, handlers[i]));
370
554
  };
371
- if (inherit)
555
+ if (inherit === true)
372
556
  signals.forEach((signal, i) => process.on(signal, handlers[i]));
373
557
  child.on('error', (error) => {
374
558
  cleanup();
@@ -380,10 +564,10 @@ export function createGitDedup(options = {}) {
380
564
  });
381
565
  });
382
566
  }
383
- async function checked(args, at = cwd) {
567
+ async function checked(args, at = cwd, progress = false, input) {
384
568
  if (hasRepositoryEnvironment)
385
569
  throw new Error('Unset Git repository override environment variables before running git-dedup storage operations');
386
- const result = await git(args, at);
570
+ const result = await git(args, at, progress && 'tee', input);
387
571
  if (result.code !== 0)
388
572
  throw new Error(`git ${args.join(' ')} failed: ${result.stderr.trim()}`);
389
573
  return result.stdout.trim();
@@ -394,7 +578,8 @@ export function createGitDedup(options = {}) {
394
578
  return canonicalPath(resolve(cwd, expandHome(configured)));
395
579
  for (const legacy of ['~/.cache/gitx', '~/.gitx']) {
396
580
  const path = expandHome(legacy);
397
- if (await isPresent(path))
581
+ const marker = await readFile(join(path, '.gitx-store'), 'utf8').catch(() => '');
582
+ if (marker === 'gitx-store-v2\n')
398
583
  return canonicalPath(path);
399
584
  }
400
585
  return canonicalPath(expandHome('~/.git-dedup'));
@@ -409,48 +594,69 @@ export function createGitDedup(options = {}) {
409
594
  // containing unrelated histories. The graph is derived data, so omit it.
410
595
  await checked(['-C', pool, 'config', 'gc.writeCommitGraph', 'false']);
411
596
  await checked(['-C', pool, 'config', 'maintenance.commit-graph.enabled', 'false']);
597
+ // Consumers borrow objects that a force push can leave unreachable in the
598
+ // pool. Automatic gc would prune them, so only store prune deletes objects,
599
+ // after re-pinning every live consumer.
600
+ await checked(['-C', pool, 'config', 'gc.auto', '0']);
601
+ await checked(['-C', pool, 'config', 'maintenance.auto', 'false']);
602
+ await checked(['-C', pool, 'config', 'gc.pruneExpire', 'never']);
412
603
  await rm(join(pool, 'objects', 'info', 'commit-graph'), { force: true });
413
604
  await rm(join(pool, 'objects', 'info', 'commit-graphs'), { recursive: true, force: true });
414
605
  return pool;
415
606
  }
416
- async function fetchRemote(root, remote, key) {
607
+ async function fetchRemote(root, remote, key, progress = false, quiet = false) {
417
608
  const pool = await ensurePool(root);
418
- await fetchRemoteObjects(pool, remote, key);
609
+ await fetchRemoteObjects(pool, remote, key, progress, quiet);
419
610
  await registerRemote(root, remote, key);
420
611
  return pool;
421
612
  }
422
- async function fetchRemoteObjects(pool, remote, key) {
613
+ async function fetchRemoteObjects(pool, remote, key, progress = false, quiet = false) {
423
614
  const id = remoteId(key);
424
- process.stderr.write(`git-dedup: updating object pool for ${key}\n`);
615
+ if (!quiet)
616
+ process.stderr.write(`git-dedup: updating object pool for ${key}\n`);
425
617
  await checked([
426
618
  '-C',
427
619
  pool,
428
620
  'fetch',
429
621
  '--no-tags',
622
+ ...(progress ? ['--progress'] : []),
430
623
  remote,
431
624
  `+refs/heads/*:refs/gitx/remotes/${id}/heads/*`,
432
625
  `+refs/tags/*:refs/gitx/remotes/${id}/tags/*`,
433
- ]);
626
+ ], cwd, progress);
434
627
  }
435
628
  async function consumerTips(repo) {
436
629
  const refs = await checked(['for-each-ref', '--format=%(objectname)', 'refs'], repo);
437
- const head = await checked(['rev-parse', 'HEAD'], repo);
438
- return [...new Set([...refs.split('\n').filter(Boolean), head])].toSorted();
630
+ // An unborn HEAD (orphan branch, bare repo naming a missing branch) has no tip to pin.
631
+ const head = await git(['rev-parse', '--verify', '-q', 'HEAD'], repo);
632
+ const tips = refs.split('\n').filter(Boolean);
633
+ if (head.code === 0)
634
+ tips.push(head.stdout.trim());
635
+ return [...new Set(tips)].toSorted();
439
636
  }
440
- async function pinnedTips(pool, id) {
441
- return new Set((await checked(['-C', pool, 'for-each-ref', '--format=%(objectname)', `refs/gitx/consumers/${id}`]))
442
- .split('\n')
443
- .filter(Boolean));
637
+ async function unpinnedTips(pool, id, tips) {
638
+ // A tip that a remote ref holds needs no pin until the remote moves. Only
639
+ // store prune deletes objects, and it re-pins every live consumer first.
640
+ const held = new Set((await checked([
641
+ '-C',
642
+ pool,
643
+ 'for-each-ref',
644
+ '--format=%(objectname)',
645
+ `refs/gitx/consumers/${id}`,
646
+ 'refs/gitx/remotes',
647
+ ])).split('\n'));
648
+ return tips.filter((oid) => !held.has(oid));
444
649
  }
445
650
  async function pinConsumer(pool, repo, commonGitdir, tips) {
446
651
  // Immutable snapshots keep old tips reachable after force pushes, branch
447
- // deletion, or a later cache call on the same checkout. Source ref names
652
+ // deletion, or a later store add call on the same checkout. Source ref names
448
653
  // cannot be copied into a files-backed pool: refs differing only by case
449
654
  // collide on case-insensitive filesystems. Object IDs are safe ref names,
450
655
  // and one pin per distinct tip preserves the same reachability.
451
- const id = await consumerId(commonGitdir);
452
- const existing = await pinnedTips(pool, id);
453
- const missing = tips.filter((oid) => !existing.has(oid));
656
+ const id = await consumerId(dirname(pool), commonGitdir);
657
+ // Register before pinning: prune refuses to run while pins lack a registration.
658
+ await registerConsumer(dirname(pool), id, commonGitdir);
659
+ const missing = await unpinnedTips(pool, id, tips);
454
660
  // Keep fetch argument lists bounded for repositories with many refs.
455
661
  for (let offset = 0; offset < missing.length; offset += 128)
456
662
  await checked([
@@ -479,39 +685,113 @@ export function createGitDedup(options = {}) {
479
685
  }
480
686
  async function clone(args, effectiveCwd) {
481
687
  const parsed = parseClone(args);
482
- if (!parsed)
483
- return undefined;
688
+ if (typeof parsed === 'string') {
689
+ fallback(args, parsed);
690
+ return plainClone(args, effectiveCwd);
691
+ }
484
692
  const key = await remoteKey(parsed.remote, effectiveCwd);
485
- if (!key)
486
- return undefined;
693
+ if (!key) {
694
+ fallback(args, 'remote is not a supported network URL');
695
+ return plainClone(args, effectiveCwd);
696
+ }
487
697
  const destination = resolve(effectiveCwd, parsed.destination ?? defaultCloneName(parsed.remote));
488
- if (await isPresent(destination))
489
- return undefined;
698
+ // Like Git, clone into a missing or empty directory.
699
+ const entries = await readdir(destination).catch((error) => error.code === 'ENOENT' ? [] : undefined);
700
+ if (!entries || entries.length)
701
+ return fallback(args, `${destination} is not an empty directory`);
490
702
  const root = await storePath();
491
- const cloned = await withLock(root, async () => {
492
- const poolReused = options.onStorageReport ? await isPresent(poolPath(root)) : false;
493
- let pool;
494
- try {
495
- pool = await fetchRemote(root, parsed.remote, key);
496
- }
497
- catch {
498
- process.stderr.write('git-dedup: object pool unavailable; using Git clone\n');
499
- return undefined;
500
- }
501
- const cloneArgs = ['clone', '--reference', pool, ...parsed.forwarded, parsed.remote, destination];
502
- const result = await git(cloneArgs, effectiveCwd, true);
503
- if (result.code !== 0)
504
- return result.code;
505
- await pinConsumer(pool, destination, await repoCommonGitdir(destination), await consumerTips(destination));
506
- if (options.onStorageReport)
507
- emitStorageReport({ operation: 'clone', repository: destination, poolReused, estimatedSavedBytes: 0 });
508
- return 0;
703
+ // Lock only the pool writes. Only store prune deletes pool objects, so git clone can
704
+ // read the pool while other processes fetch or repack, and the new tips are pinned right after.
705
+ let poolReused = false;
706
+ let fetched;
707
+ let fetchError;
708
+ // Like git clone: progress on a terminal or with --progress, never with -q/--quiet.
709
+ const quiet = parsed.forwarded.some((arg) => arg === '-q' || arg === '--quiet');
710
+ const progress = !quiet && (process.stderr.isTTY || parsed.forwarded.includes('--progress'));
711
+ // Until the pin lands, this clone borrows pool objects that nothing pins, so
712
+ // store prune refuses while the marker exists. A failed pin leaves it behind.
713
+ const marker = join(root, 'clones', `${randomUUID()}.json`);
714
+ try {
715
+ fetched = await withLock(root, async () => {
716
+ poolReused = options.onStorageReport ? await isPresent(poolPath(root)) : false;
717
+ const pool = await fetchRemote(root, parsed.remote, key, progress, quiet).catch((error) => {
718
+ fetchError = error;
719
+ return undefined;
720
+ });
721
+ if (pool) {
722
+ await mkdir(dirname(marker), { recursive: true });
723
+ await writeFile(marker, JSON.stringify({ pid: process.pid, destination }) + '\n');
724
+ }
725
+ return pool;
726
+ });
727
+ }
728
+ catch {
729
+ return fallback(args, 'object pool lock unavailable');
730
+ }
731
+ if (!fetched)
732
+ return fallback(args, `object pool unavailable${gitFailure(fetchError)}`);
733
+ const pool = fetched;
734
+ const cloneArgs = ['clone', '--reference', pool, ...parsed.forwarded, parsed.remote, destination];
735
+ const result = await git(cloneArgs, effectiveCwd, true);
736
+ if (result.code !== 0) {
737
+ if (!(await isPresent(join(destination, '.git'))))
738
+ await rm(marker, { force: true });
739
+ return result.code;
740
+ }
741
+ const commonGitdir = await repoCommonGitdir(destination);
742
+ const tips = await consumerTips(destination);
743
+ await withLock(root, async () => {
744
+ await pinConsumer(pool, destination, commonGitdir, tips);
745
+ await rm(marker, { force: true });
509
746
  });
510
- if (cloned === 0 && parsed.recurse)
511
- return updateSubmodules(['update', '--init', '--recursive'], destination);
512
- return cloned;
747
+ if (options.onStorageReport)
748
+ emitStorageReport({ operation: 'clone', repository: destination, poolReused, estimatedSavedBytes: 0 });
749
+ if (!parsed.recurse)
750
+ return 0;
751
+ // Like native clone, activate every submodule, including ones added upstream later.
752
+ const active = await git(['config', 'submodule.active', '.'], destination, true);
753
+ if (active.code !== 0)
754
+ return active.code;
755
+ return updateSubmodules(['update', '--init', '--recursive', ...(quiet ? ['-q'] : [])], destination);
513
756
  }
514
- async function seedSubmodules(repo) {
757
+ /**
758
+ * Runs `git clone` unchanged, then pins the new repository if it borrows from the pool:
759
+ * a local clone of a linked checkout copies its alternates.
760
+ */
761
+ async function plainClone(args, at) {
762
+ // Rather than re-parse Git's options, try each operand as the directory and as Git's guess from it
763
+ // (`.git` appended for --bare/--mirror). Git only clones into a missing or empty directory.
764
+ const candidates = [];
765
+ for (const operand of args.filter((arg, i) => !arg.startsWith('-') || args.slice(0, i).includes('--')))
766
+ for (const path of [operand, defaultCloneName(operand), `${defaultCloneName(operand)}.git`]) {
767
+ const candidate = resolve(at, path);
768
+ if (!(await readdir(candidate).catch(() => [])).length)
769
+ candidates.push(candidate);
770
+ }
771
+ const result = await git(['clone', ...args], at, true);
772
+ if (result.code === 0)
773
+ for (const candidate of new Set(candidates))
774
+ await pinBorrower(candidate);
775
+ return result.code;
776
+ }
777
+ async function pinBorrower(destination) {
778
+ // A worktree (or --separate-git-dir gitfile) at the destination, or a bare/mirror gitdir that is the destination.
779
+ if (!(await isPresent(join(destination, '.git')))) {
780
+ const gitdir = await git(['rev-parse', '--absolute-git-dir'], destination).catch(() => undefined);
781
+ if (gitdir?.code !== 0 || gitdir.stdout.trim() !== (await canonicalPath(destination)))
782
+ return;
783
+ }
784
+ const root = await storePath();
785
+ const pool = poolPath(root);
786
+ const commonGitdir = await repoCommonGitdir(destination);
787
+ const alternates = await readFile(join(commonGitdir, 'objects', 'info', 'alternates'), 'utf8').catch(() => '');
788
+ if (!alternateLines(alternates).includes(join(pool, 'objects')))
789
+ return;
790
+ // The source's pins hold these objects until this pin lands.
791
+ const tips = await consumerTips(destination);
792
+ await withLock(root, () => pinConsumer(pool, destination, commonGitdir, tips));
793
+ }
794
+ async function seedSubmodules(repo, quiet = false) {
515
795
  const modulesFile = join(repo, '.gitmodules');
516
796
  if (!(await isPresent(modulesFile)))
517
797
  return;
@@ -542,7 +822,8 @@ export function createGitDedup(options = {}) {
542
822
  if (!remote || !key)
543
823
  continue;
544
824
  const target = resolve(repo, item.path);
545
- if (!target.startsWith(repo + sep) || (await isPresent(join(target, '.git'))))
825
+ // Leave populated paths (an initialized checkout or user files) to native Git, which refuses to clobber them.
826
+ if (!target.startsWith(repo + sep) || (await readdir(target).catch(() => [])).length > 0)
546
827
  continue;
547
828
  const gitPathResult = await git(['rev-parse', '--path-format=absolute', '--git-path', `modules/${name}`], repo);
548
829
  if (gitPathResult.code !== 0)
@@ -552,9 +833,24 @@ export function createGitDedup(options = {}) {
552
833
  continue;
553
834
  const root = await storePath();
554
835
  await withLock(root, async () => {
555
- const pool = await fetchRemote(root, remote, key);
836
+ const pool = await fetchRemote(root, remote, key, false, quiet);
556
837
  await mkdir(dirname(gitdir), { recursive: true });
557
838
  await checked(['clone', '--bare', '--reference', pool, remote, gitdir]);
839
+ // Match a native submodule clone: remote-tracking refs, origin/HEAD, and only the default local branch.
840
+ await checked(['--git-dir', gitdir, 'config', 'remote.origin.fetch', '+refs/heads/*:refs/remotes/origin/*']);
841
+ await checked(['--git-dir', gitdir, 'fetch', '--quiet', 'origin']);
842
+ const head = await git(['--git-dir', gitdir, 'symbolic-ref', '--short', '-q', 'HEAD']);
843
+ const branch = head.code === 0 ? head.stdout.trim() : undefined;
844
+ if (branch) {
845
+ await checked(['--git-dir', gitdir, 'remote', 'set-head', 'origin', branch]);
846
+ await checked(['--git-dir', gitdir, 'config', `branch.${branch}.remote`, 'origin']);
847
+ await checked(['--git-dir', gitdir, 'config', `branch.${branch}.merge`, `refs/heads/${branch}`]);
848
+ }
849
+ const others = (await checked(['--git-dir', gitdir, 'for-each-ref', '--format=%(refname:short)', 'refs/heads']))
850
+ .split('\n')
851
+ .filter((ref) => ref && ref !== branch);
852
+ if (others.length > 0)
853
+ await checked(['--git-dir', gitdir, 'branch', '-D', ...others]);
558
854
  await checked(['--git-dir', gitdir, 'config', 'core.bare', 'false']);
559
855
  await checked(['--git-dir', gitdir, 'config', 'core.worktree', target]);
560
856
  await pinConsumer(pool, gitdir, gitdir, await consumerTips(gitdir));
@@ -562,35 +858,50 @@ export function createGitDedup(options = {}) {
562
858
  }
563
859
  }
564
860
  async function updateSubmodules(args, at) {
861
+ const quiet = args.includes('-q') || args.includes('--quiet');
565
862
  if (args[0] === 'add') {
566
863
  const code = (await git(['submodule', ...args], at, true)).code;
567
864
  if (code !== 0)
568
865
  return code;
569
- const target = args.length === 2 ? defaultCloneName(args[1]) : args.length === 3 ? args[2] : undefined;
570
- if (target && !target.startsWith('-')) {
866
+ const target = submoduleAddPath(args);
867
+ if (!target)
868
+ fallback(args, 'submodule path not recognized');
869
+ else {
571
870
  try {
572
- await cache(resolve(at, target));
871
+ await add(resolve(at, target), quiet);
573
872
  }
574
873
  catch (error) {
575
- process.stderr.write(`git-dedup: submodule cache unavailable (${String(error)})\n`);
874
+ process.stderr.write(`git-dedup: submodule adoption unavailable (${String(error)})\n`);
576
875
  }
577
876
  }
578
877
  return 0;
579
878
  }
580
- if (args[0] !== 'update' || args.slice(1).some((arg) => !['--init', '--recursive', '--quiet', '-q'].includes(arg)))
879
+ if (args[0] !== 'update')
581
880
  return (await git(['submodule', ...args], at, true)).code;
582
- if ((await git(['config', '--get', 'submodule.active'], at)).code === 0)
881
+ // Without a pathspec, `git submodule update` from a subdirectory covers the whole superproject.
882
+ const toplevel = await git(['rev-parse', '--show-toplevel'], at);
883
+ const repo = toplevel.code === 0 ? toplevel.stdout.trim() : at;
884
+ const unsupported = args.slice(1).find((arg) => !['--init', '--recursive', '--quiet', '-q'].includes(arg));
885
+ if (unsupported) {
886
+ fallback(args, `submodule update ${unsupported} is not supported`);
583
887
  return (await git(['submodule', ...args], at, true)).code;
888
+ }
889
+ // The native clone default '.' activates every submodule, which seeding already handles.
890
+ const active = await git(['config', '--get-all', 'submodule.active'], at);
891
+ if (active.code === 0 && active.stdout.trim() !== '.') {
892
+ fallback(args, 'submodule.active is configured');
893
+ return (await git(['submodule', ...args], at, true)).code;
894
+ }
584
895
  if (args.includes('--init')) {
585
- const init = (await git(['submodule', 'init'], at, true)).code;
896
+ const init = (await git(['submodule', 'init', ...(quiet ? ['-q'] : [])], at, true)).code;
586
897
  if (init !== 0)
587
898
  return init;
588
899
  }
589
900
  try {
590
- await seedSubmodules(at);
901
+ await seedSubmodules(repo, quiet);
591
902
  }
592
- catch {
593
- process.stderr.write('git-dedup: submodule cache unavailable; using Git\n');
903
+ catch (error) {
904
+ fallback(args, `submodule adoption unavailable${gitFailure(error)}`);
594
905
  }
595
906
  const recursive = args.includes('--recursive') &&
596
907
  args.includes('--init') &&
@@ -600,24 +911,39 @@ export function createGitDedup(options = {}) {
600
911
  const top = (await git(['submodule', ...args.filter((arg) => arg !== '--recursive')], at, true)).code;
601
912
  if (top !== 0)
602
913
  return top;
603
- const listing = await git(['config', '--file', '.gitmodules', '--get-regexp', '^submodule\\..*\\.path$'], at);
914
+ const listing = await git(['config', '--file', '.gitmodules', '--get-regexp', '^submodule\\..*\\.path$'], repo);
604
915
  if (listing.code !== 0)
605
916
  return 0;
606
917
  for (const line of listing.stdout.trim().split('\n')) {
607
918
  const match = line.match(/^submodule\..*\.path (.+)$/);
608
919
  if (!match)
609
920
  continue;
610
- const child = resolve(at, match[1]);
611
- if (!child.startsWith(at + sep) || !(await isPresent(child)))
921
+ const child = resolve(repo, match[1]);
922
+ if (!child.startsWith(repo + sep) || !(await isPresent(child)))
612
923
  continue;
613
- const nested = await updateSubmodules(['update', '--init', '--recursive'], child);
924
+ const nested = await updateSubmodules(['update', '--init', '--recursive', ...(quiet ? ['-q'] : [])], child);
614
925
  if (nested !== 0)
615
926
  return nested;
616
927
  }
617
928
  return 0;
618
929
  }
619
- async function cache(path = cwd) {
620
- const result = { cached: 0, skipped: 0, failed: 0, repositories: [] };
930
+ /** Visit the checkout of every initialized submodule gitdir under a modules directory. */
931
+ async function visitModules(directory, visit) {
932
+ for (const entry of await readdir(directory, { withFileTypes: true }).catch(() => [])) {
933
+ if (!entry.isDirectory())
934
+ continue;
935
+ const modulePath = join(directory, entry.name);
936
+ if (await isPresent(join(modulePath, 'HEAD'))) {
937
+ const workTree = (await git(['--git-dir', modulePath, 'config', '--get', 'core.worktree'])).stdout.trim();
938
+ if (workTree)
939
+ await visit(resolve(modulePath, workTree));
940
+ }
941
+ else
942
+ await visitModules(modulePath, visit);
943
+ }
944
+ }
945
+ async function add(path = cwd, quiet = false) {
946
+ const result = { added: 0, skipped: 0, failed: 0, repositories: [] };
621
947
  const visited = new Set();
622
948
  function record(repoPath, status, reason, detail) {
623
949
  result[status]++;
@@ -668,32 +994,47 @@ export function createGitDedup(options = {}) {
668
994
  const marker = join(gitdir, 'gitx-cache.json');
669
995
  const state = JSON.stringify({ version: 1, pool, remote: url, key, tips }) + '\n';
670
996
  const alternate = join(commonGitdir, 'objects', 'info', 'alternates');
671
- const alreadyLinked = (await readFile(alternate, 'utf8').catch(() => ''))
672
- .split('\n')
673
- .includes(join(pool, 'objects'));
997
+ const originalAlternates = await readFile(alternate, 'utf8').catch(() => undefined);
998
+ const alreadyLinked = alternateLines(originalAlternates ?? '').includes(join(pool, 'objects'));
674
999
  const registered = (await readFile(join(root, 'remotes', `${remoteId(key)}.json`), 'utf8').catch(() => '')) ===
675
- JSON.stringify({ key, remote: url }) + '\n';
1000
+ registration(url, key);
676
1001
  const stateMatches = poolReused && alreadyLinked && registered && (await readFile(marker, 'utf8').catch(() => '')) === state;
677
- const currentPins = stateMatches ? await pinnedTips(pool, await consumerId(commonGitdir)) : undefined;
678
- if (currentPins && tips.every((oid) => currentPins.has(oid))) {
1002
+ const id = await consumerId(root, commonGitdir);
1003
+ if (stateMatches && !(await unpinnedTips(pool, id, tips)).length) {
1004
+ // Checkouts adopted before the consumer registry existed register here.
1005
+ await registerConsumer(root, id, commonGitdir);
679
1006
  record(repo, 'skipped', 'already current');
680
- process.stderr.write(`git-dedup: caching ${repo}: already current\n`);
1007
+ if (!quiet)
1008
+ process.stderr.write(`git-dedup: adding ${repo}: already current\n`);
681
1009
  return;
682
1010
  }
683
- process.stderr.write(`git-dedup: caching ${repo}: packing local objects\n`);
1011
+ if (!quiet)
1012
+ process.stderr.write(`git-dedup: adding ${repo}: packing local objects\n`);
684
1013
  // Gather loose local objects before adding an alternate; afterwards Git may
685
1014
  // consider a local object redundant and omit it from a new pack.
686
1015
  if (!alreadyLinked)
687
1016
  await checked(['-c', 'repack.writeBitmaps=false', 'repack', '-a', '-d', '-l'], repo);
688
1017
  const beforeUniqueBytes = await uniquePackBytes(commonGitdir);
689
- process.stderr.write(`git-dedup: caching ${repo}: importing refs\n`);
1018
+ if (!quiet)
1019
+ process.stderr.write(`git-dedup: adding ${repo}: importing refs\n`);
690
1020
  stage = 'importing refs';
691
1021
  await pinConsumer(pool, repo, commonGitdir, tips);
692
1022
  stage = 'sharing objects';
693
- process.stderr.write(`git-dedup: caching ${repo}: sharing objects\n`);
694
- await setAlternate(commonGitdir, pool);
695
- await removeLegacyKeeps(commonGitdir);
696
- await checked(['-c', 'repack.writeBitmaps=false', 'repack', '-a', '-d', '-l'], repo);
1023
+ if (!quiet)
1024
+ process.stderr.write(`git-dedup: adding ${repo}: sharing objects\n`);
1025
+ try {
1026
+ await setAlternate(commonGitdir, pool);
1027
+ await removeLegacyKeeps(commonGitdir);
1028
+ await checked(['-c', 'repack.writeBitmaps=false', 'repack', '-a', '-d', '-l'], repo);
1029
+ }
1030
+ catch (error) {
1031
+ // Restore only until this repack succeeds; afterwards objects it dropped live only in the pool.
1032
+ if (originalAlternates === undefined)
1033
+ await rm(alternate, { force: true });
1034
+ else
1035
+ await writeFile(alternate, originalAlternates);
1036
+ throw error;
1037
+ }
697
1038
  if ((await isPresent(join(commonGitdir, 'objects', 'info', 'commit-graph'))) ||
698
1039
  (await isPresent(join(commonGitdir, 'objects', 'info', 'commit-graphs'))))
699
1040
  await checked(['commit-graph', 'write', '--reachable'], repo);
@@ -702,7 +1043,7 @@ export function createGitDedup(options = {}) {
702
1043
  await registerRemote(root, url, key);
703
1044
  let refreshError;
704
1045
  try {
705
- await fetchRemoteObjects(pool, url, key);
1046
+ await fetchRemoteObjects(pool, url, key, false, quiet);
706
1047
  }
707
1048
  catch (error) {
708
1049
  refreshError = error;
@@ -711,7 +1052,7 @@ export function createGitDedup(options = {}) {
711
1052
  if (options.onStorageReport) {
712
1053
  const afterUniqueBytes = await uniquePackBytes(commonGitdir);
713
1054
  emitStorageReport({
714
- operation: 'cache',
1055
+ operation: 'add',
715
1056
  repository: repo,
716
1057
  poolReused,
717
1058
  beforeUniqueBytes,
@@ -719,33 +1060,132 @@ export function createGitDedup(options = {}) {
719
1060
  estimatedSavedBytes: Math.max(0, beforeUniqueBytes - afterUniqueBytes),
720
1061
  });
721
1062
  }
722
- record(repo, 'cached', refreshError ? 'remote refresh deferred; retry with git-dedup store fetch' : undefined, refreshError ? String(refreshError) : undefined);
1063
+ record(repo, 'added', refreshError ? 'remote refresh deferred; retry with git-dedup store fetch' : undefined, refreshError ? String(refreshError) : undefined);
723
1064
  if (refreshError)
724
- process.stderr.write(`git-dedup: caching ${repo}: remote refresh deferred; retry with git-dedup store fetch\n`);
1065
+ process.stderr.write(`git-dedup: adding ${repo}: remote refresh deferred; retry with git-dedup store fetch\n`);
725
1066
  });
726
1067
  }
727
1068
  catch (error) {
728
- record(repo, 'failed', cacheFailureReason(error, stage), String(error));
1069
+ record(repo, 'failed', addFailureReason(error, stage), String(error));
729
1070
  }
730
1071
  finally {
731
- await visitModules(join(gitdir, 'modules'));
1072
+ await visitModules(join(gitdir, 'modules'), adopt);
732
1073
  }
733
- async function visitModules(directory) {
734
- for (const entry of await readdir(directory, { withFileTypes: true }).catch(() => [])) {
735
- if (!entry.isDirectory())
736
- continue;
737
- const modulePath = join(directory, entry.name);
738
- if (await isPresent(join(modulePath, 'HEAD'))) {
739
- const workTree = (await git(['--git-dir', modulePath, 'config', '--get', 'core.worktree'])).stdout.trim();
740
- if (workTree)
741
- await adopt(resolve(modulePath, workTree));
742
- }
743
- else
744
- await visitModules(modulePath);
1074
+ }
1075
+ await adopt(resolve(cwd, path));
1076
+ return result;
1077
+ }
1078
+ async function remove(path = cwd) {
1079
+ const result = { removed: 0, skipped: 0, failed: 0, repositories: [] };
1080
+ const visited = new Set();
1081
+ const root = await storePath();
1082
+ const pool = poolPath(root);
1083
+ const poolObjects = join(pool, 'objects');
1084
+ function record(entry) {
1085
+ result[entry.status]++;
1086
+ result.repositories.push(entry);
1087
+ }
1088
+ async function detach(repo) {
1089
+ let commonGitdir;
1090
+ try {
1091
+ commonGitdir = await repoCommonGitdir(repo);
1092
+ }
1093
+ catch (error) {
1094
+ const missing = !(await isPresent(repo));
1095
+ record({
1096
+ path: repo,
1097
+ status: missing ? 'failed' : 'skipped',
1098
+ reason: missing
1099
+ ? `path not found; if this checkout was deleted, run git-dedup store remove --forget ${repo}`
1100
+ : 'not a Git repository or path is unavailable',
1101
+ detail: String(error),
1102
+ });
1103
+ return;
1104
+ }
1105
+ // Linked worktrees share one object database, so detach it once.
1106
+ if (visited.has(commonGitdir))
1107
+ return;
1108
+ visited.add(commonGitdir);
1109
+ const gitdirs = [
1110
+ commonGitdir,
1111
+ ...(await readdir(join(commonGitdir, 'worktrees')).catch(() => [])).map((name) => join(commonGitdir, 'worktrees', name)),
1112
+ ];
1113
+ const alternate = join(commonGitdir, 'objects', 'info', 'alternates');
1114
+ try {
1115
+ const original = await readFile(alternate, 'utf8').catch(() => '');
1116
+ const lines = alternateLines(original);
1117
+ if (!lines.includes(poolObjects)) {
1118
+ record({ path: repo, status: 'skipped', reason: 'not linked to the store' });
1119
+ return;
745
1120
  }
1121
+ if (!(await isPresent(pool)))
1122
+ throw new Error(`Object pool is missing: ${pool}`);
1123
+ // Lock without initializing: removal must never create a store.
1124
+ await lockDirectory(storeLock(root), async () => {
1125
+ process.stderr.write(`git-dedup: removing ${repo}: copying shared objects\n`);
1126
+ // Repack and fsck ignore in-progress merge, cherry-pick, and rebase state, so pin the
1127
+ // objects it names with temporary refs. Names that are already missing stay missing.
1128
+ const named = new Set();
1129
+ for (const gitdir of gitdirs)
1130
+ for (const oid of await stateOids(gitdir))
1131
+ named.add(oid);
1132
+ const present = named.size
1133
+ ? (await checked(['cat-file', '--batch-check=%(objectname)'], repo, false, [...named].join('\n') + '\n'))
1134
+ .split('\n')
1135
+ .filter((line) => /^[0-9a-f]{40}$/.test(line))
1136
+ : [];
1137
+ try {
1138
+ if (present.length)
1139
+ await checked(['update-ref', '--stdin'], repo, false, present.map((oid) => `update refs/gitx-detach/${oid} ${oid}\n`).join(''));
1140
+ // Without -l, repack copies every object reachable from refs, reflogs, and
1141
+ // the indexes of all worktrees, including objects read through the pool.
1142
+ await checked(['-c', 'repack.writeBitmaps=false', 'repack', '-a', '-d', '--pack-kept-objects'], repo);
1143
+ const remaining = lines.filter((line) => line !== poolObjects);
1144
+ if (remaining.length)
1145
+ await writeFile(alternate, remaining.join('\n') + '\n');
1146
+ else
1147
+ await rm(alternate);
1148
+ try {
1149
+ await checked(['fsck', '--connectivity-only'], repo);
1150
+ }
1151
+ catch (error) {
1152
+ await writeFile(alternate, original);
1153
+ throw error;
1154
+ }
1155
+ }
1156
+ finally {
1157
+ if (present.length)
1158
+ await checked(['update-ref', '--stdin'], repo, false, present.map((oid) => `delete refs/gitx-detach/${oid}\n`).join(''));
1159
+ }
1160
+ const id = (await readFile(join(commonGitdir, 'gitx-consumer-id'), 'utf8').catch(() => '')).trim();
1161
+ // A copy carrying another live checkout's ID detaches without releasing that checkout's pins.
1162
+ if (/^[a-f0-9-]{36}$/.test(id) && !(await ownedElsewhere(root, id, commonGitdir))) {
1163
+ const pins = await checked([
1164
+ '-C',
1165
+ pool,
1166
+ 'for-each-ref',
1167
+ '--format=delete %(refname)',
1168
+ `refs/gitx/consumers/${id}`,
1169
+ ]);
1170
+ if (pins)
1171
+ await checked(['-C', pool, 'update-ref', '--stdin'], cwd, false, pins + '\n');
1172
+ await rm(join(root, 'consumers', `${id}.json`), { force: true });
1173
+ }
1174
+ await rm(join(commonGitdir, 'gitx-consumer-id'), { force: true });
1175
+ for (const gitdir of gitdirs)
1176
+ await rm(join(gitdir, 'gitx-cache.json'), { force: true });
1177
+ });
1178
+ record({ path: repo, status: 'removed', objectBytes: await directorySize(join(commonGitdir, 'objects')) });
1179
+ }
1180
+ catch (error) {
1181
+ record({ path: repo, status: 'failed', reason: 'could not detach from the store', detail: String(error) });
1182
+ }
1183
+ finally {
1184
+ for (const gitdir of gitdirs)
1185
+ await visitModules(join(gitdir, 'modules'), detach);
746
1186
  }
747
1187
  }
748
- await adopt(resolve(cwd, path));
1188
+ await detach(resolve(cwd, path));
749
1189
  return result;
750
1190
  }
751
1191
  async function storeInfo() {
@@ -776,25 +1216,33 @@ export function createGitDedup(options = {}) {
776
1216
  typeof value.remote !== 'string' ||
777
1217
  `${remoteId(value.key)}.json` !== name)
778
1218
  throw new Error(`Invalid git-dedup remote registration: ${name}`);
779
- return { key: value.key, remote: value.remote };
1219
+ return { key: value.key, remote: redactRemote(value.remote, '***') };
780
1220
  }));
781
1221
  return remotes.toSorted((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
782
1222
  }
783
1223
  async function fetch() {
784
1224
  const root = await storePath();
785
- let fetched = 0;
1225
+ const remotes = [];
786
1226
  await withLock(root, async () => {
787
1227
  for (const name of await readdir(join(root, 'remotes')).catch(() => [])) {
788
1228
  if (!name.endsWith('.json'))
789
1229
  continue;
790
1230
  const { key, remote } = JSON.parse(await readFile(join(root, 'remotes', name), 'utf8'));
791
- await fetchRemote(root, remote, key);
792
- fetched++;
1231
+ try {
1232
+ await fetchRemote(root, remote, key);
1233
+ remotes.push({ key, status: 'fetched' });
1234
+ }
1235
+ catch (error) {
1236
+ const detail = String(error);
1237
+ const reason = /fatal: (.+)/.exec(detail)?.[1] ?? 'remote fetch failed';
1238
+ remotes.push({ key, status: 'failed', reason, detail });
1239
+ }
793
1240
  }
794
- if (fetched)
1241
+ if (remotes.some((result) => result.status === 'fetched'))
795
1242
  await checked(['-C', poolPath(root), 'repack', '-d', '--geometric=2']);
796
1243
  });
797
- return { fetched };
1244
+ const fetched = remotes.filter((result) => result.status === 'fetched').length;
1245
+ return { fetched, failed: remotes.length - fetched, remotes };
798
1246
  }
799
1247
  async function gc() {
800
1248
  const root = await storePath();
@@ -805,26 +1253,178 @@ export function createGitDedup(options = {}) {
805
1253
  });
806
1254
  return { compacted: true };
807
1255
  }
808
- async function doctor() {
1256
+ /** Every object a consumer may borrow from the pool without a pin. */
1257
+ async function liveTips(commonGitdir) {
1258
+ if ((await git(['--git-dir', commonGitdir, 'config', '--get', 'extensions.refStorage'], commonGitdir)).stdout.trim())
1259
+ throw new Error(`Refusing to prune: ${commonGitdir} uses a ref storage format that git-dedup cannot inspect`);
1260
+ const tips = new Set();
1261
+ const extra = new Set();
1262
+ const worktrees = (await readdir(join(commonGitdir, 'worktrees')).catch(() => [])).map((name) => join(commonGitdir, 'worktrees', name));
1263
+ for (const gitdir of [commonGitdir, ...worktrees]) {
1264
+ // Per-worktree refs and HEAD, then the index: git add skips blobs the pool already has.
1265
+ for (const oid of (await checked(['--git-dir', gitdir, 'for-each-ref', '--format=%(objectname)'], gitdir)).split('\n'))
1266
+ if (oid)
1267
+ tips.add(oid);
1268
+ const head = await git(['--git-dir', gitdir, 'rev-parse', '--verify', '-q', 'HEAD'], gitdir);
1269
+ if (head.code === 0)
1270
+ tips.add(head.stdout.trim());
1271
+ if (await isPresent(join(gitdir, 'index'))) {
1272
+ const tree = await git(['--git-dir', gitdir, 'write-tree'], gitdir);
1273
+ if (tree.code !== 0)
1274
+ throw new Error(`Refusing to prune: cannot record the index of ${gitdir}: ${tree.stderr.trim()}`);
1275
+ // Git always has the empty tree, so it needs no pin.
1276
+ if (tree.stdout.trim() !== '4b825dc642cb6eb9a060e54bf8d69288fbee4904')
1277
+ tips.add(tree.stdout.trim());
1278
+ }
1279
+ for (const oid of await stateOids(gitdir))
1280
+ if (!tips.has(oid))
1281
+ extra.add(oid);
1282
+ }
1283
+ // rev-list peels annotated tags to commits, so pin state-file tags (and other non-commits) directly.
1284
+ if (extra.size)
1285
+ for (const line of (await checked(['--git-dir', commonGitdir, 'cat-file', '--batch-check=%(objectname) %(objecttype)'], commonGitdir, false, [...extra].join('\n') + '\n')).split('\n')) {
1286
+ const [oid, type] = line.split(' ');
1287
+ if (type && type !== 'commit' && type !== 'missing')
1288
+ tips.add(oid);
1289
+ }
1290
+ // Commits held only by reflogs or state files; pin the tips of each abandoned history.
1291
+ const abandoned = await checked(['--git-dir', commonGitdir, 'rev-list', '--parents', '--ignore-missing', '--reflog', ...extra, '--not', '--all'], commonGitdir);
1292
+ const commits = new Set();
1293
+ const parents = new Set();
1294
+ for (const line of abandoned.split('\n').filter(Boolean)) {
1295
+ const [commit, ...rest] = line.split(' ');
1296
+ commits.add(commit);
1297
+ for (const parent of rest)
1298
+ parents.add(parent);
1299
+ }
1300
+ for (const commit of commits)
1301
+ if (!parents.has(commit))
1302
+ tips.add(commit);
1303
+ return [...tips].toSorted();
1304
+ }
1305
+ async function prune() {
809
1306
  const root = await storePath();
1307
+ const pool = poolPath(root);
1308
+ if (!(await isPresent(pool)))
1309
+ return { reclaimedBytes: 0 };
810
1310
  return withLock(root, async () => {
811
- const checks = [];
812
- const pool = await ensurePool(root);
813
- checks.push({
814
- name: 'pool',
815
- ok: true,
816
- detail: `${pool}: SHA-1 object database`,
817
- });
1311
+ const registry = await registeredConsumers(root);
1312
+ // A clone between its pool fetch and its pin borrows objects that nothing pins yet.
1313
+ for (const name of await readdir(join(root, 'clones')).catch(() => [])) {
1314
+ const { pid, destination } = JSON.parse(await readFile(join(root, 'clones', name), 'utf8'));
1315
+ if (processExists(pid))
1316
+ throw new Error(`Refusing to prune: a clone into ${destination} is in progress; retry when it finishes.`);
1317
+ const id = (await readFile(join(destination, '.git', 'gitx-consumer-id'), 'utf8').catch(() => '')).trim();
1318
+ if ((await isPresent(join(destination, '.git'))) && !registry.has(id))
1319
+ throw new Error(`Refusing to prune: the clone into ${destination} was never registered. ` +
1320
+ `Run git-dedup store add ${destination}, or delete that checkout, then retry.`);
1321
+ await rm(join(root, 'clones', name));
1322
+ }
1323
+ const pinned = new Set((await checked(['-C', pool, 'for-each-ref', '--format=%(refname)', 'refs/gitx/consumers']))
1324
+ .split('\n')
1325
+ .filter(Boolean)
1326
+ .map((ref) => ref.split('/')[3]));
1327
+ const unregistered = [...pinned].filter((id) => !registry.has(id));
1328
+ if (unregistered.length)
1329
+ throw new Error(`Refusing to prune: the pool holds objects for ${unregistered.length} unregistered checkout(s). ` +
1330
+ 'Run git-dedup store add for every checkout that uses this store (for example ' +
1331
+ 'git-dedup store add --all <directory>), then retry.');
1332
+ // A moved checkout still borrows through its absolute alternate path, including
1333
+ // objects fetched after its last pin. Only the user can say it was deleted.
1334
+ const missing = [];
1335
+ for (const [id, gitdir] of registry)
1336
+ if (!(await carriesConsumerId(gitdir, id)))
1337
+ missing.push(gitdir);
1338
+ if (missing.length)
1339
+ throw new Error(`Refusing to prune: ${missing.length} registered checkout(s) cannot be found:\n` +
1340
+ missing.map((gitdir) => ` ${checkoutPath(gitdir)}\n`).join('') +
1341
+ 'If this checkout was moved, run git-dedup store add <new path>. ' +
1342
+ 'If it was deleted, run git-dedup store remove --forget <old path>.');
1343
+ for (const gitdir of registry.values())
1344
+ await pinConsumer(pool, gitdir, gitdir, await liveTips(gitdir));
1345
+ const before = await directorySize(pool);
1346
+ // Explicit expiry: the pool never prunes automatically.
1347
+ await checked(['-C', pool, 'gc', '--prune=now']);
1348
+ return { reclaimedBytes: Math.max(0, before - (await directorySize(pool))) };
1349
+ });
1350
+ }
1351
+ /** Release a registered checkout that no longer exists. Returns the Git directories forgotten. */
1352
+ async function forget(path) {
1353
+ const root = await storePath();
1354
+ const target = resolve(cwd, path);
1355
+ const candidates = new Set();
1356
+ for (const base of [target, await canonicalPath(target)])
1357
+ candidates.add(base).add(join(base, '.git'));
1358
+ if (!(await isPresent(root)))
1359
+ return [];
1360
+ return lockDirectory(storeLock(root), async () => {
1361
+ const forgotten = [];
1362
+ let present = false;
1363
+ for (const [id, gitdir] of await registeredConsumers(root)) {
1364
+ if (!candidates.has(gitdir))
1365
+ continue;
1366
+ // A new checkout at a deleted checkout's path is live; forget only the old registration.
1367
+ if (await carriesConsumerId(gitdir, id)) {
1368
+ present = true;
1369
+ continue;
1370
+ }
1371
+ if (await isPresent(poolPath(root))) {
1372
+ const pins = await checked([
1373
+ '-C',
1374
+ poolPath(root),
1375
+ 'for-each-ref',
1376
+ '--format=delete %(refname)',
1377
+ `refs/gitx/consumers/${id}`,
1378
+ ]);
1379
+ if (pins)
1380
+ await checked(['-C', poolPath(root), 'update-ref', '--stdin'], cwd, false, pins + '\n');
1381
+ }
1382
+ await rm(join(root, 'consumers', `${id}.json`));
1383
+ forgotten.push(gitdir);
1384
+ }
1385
+ if (!forgotten.length && present)
1386
+ throw new Error(`${path} still exists; run git-dedup store remove without --forget to detach it`);
1387
+ return forgotten;
1388
+ });
1389
+ }
1390
+ async function doctor() {
1391
+ const root = await storePath();
1392
+ const checks = [];
1393
+ const pool = poolPath(root);
1394
+ const empty = !(await isPresent(pool));
1395
+ if (!empty) {
1396
+ try {
1397
+ const format = await checked(['-C', pool, 'rev-parse', '--show-object-format']);
1398
+ const bare = await checked(['-C', pool, 'rev-parse', '--is-bare-repository']);
1399
+ checks.push({
1400
+ name: 'pool',
1401
+ ok: format === 'sha1' && bare === 'true',
1402
+ detail: `${pool}: ${format === 'sha1' ? 'SHA-1' : format.toUpperCase()} ${bare === 'true' ? 'bare' : 'non-bare'} object database`,
1403
+ });
1404
+ }
1405
+ catch (error) {
1406
+ checks.push({
1407
+ name: 'pool',
1408
+ ok: false,
1409
+ detail: `${pool}: ${error instanceof Error ? error.message : String(error)}`,
1410
+ });
1411
+ }
1412
+ }
1413
+ try {
818
1414
  const binary = await gitPath();
819
1415
  checks.push({ name: 'git', ok: true, detail: `${binary}: ${await checked(['--version'])}` });
820
- return { checks };
821
- });
1416
+ }
1417
+ catch (error) {
1418
+ checks.push({ name: 'git', ok: false, detail: error instanceof Error ? error.message : String(error) });
1419
+ }
1420
+ return { empty, checks };
822
1421
  }
823
1422
  async function addWorktree(args, at) {
824
1423
  if (args[0] !== 'add')
825
1424
  return (await git(['worktree', ...args], at, true)).code;
826
1425
  let path;
827
1426
  let noCheckout = false;
1427
+ let quiet = false;
828
1428
  for (let i = 1; i < args.length; i++) {
829
1429
  const arg = args[i];
830
1430
  if (arg === '--') {
@@ -837,6 +1437,8 @@ export function createGitDedup(options = {}) {
837
1437
  }
838
1438
  if (arg === '--no-checkout' || arg === '--orphan')
839
1439
  noCheckout = true;
1440
+ if (arg === '-q' || arg === '--quiet')
1441
+ quiet = true;
840
1442
  if ([
841
1443
  '-f',
842
1444
  '--force',
@@ -855,8 +1457,10 @@ export function createGitDedup(options = {}) {
855
1457
  ].includes(arg) ||
856
1458
  /^(--reason|--track)=/.test(arg))
857
1459
  continue;
858
- if (arg.startsWith('-'))
1460
+ if (arg.startsWith('-')) {
1461
+ fallback(args, `worktree add ${arg.split('=')[0]} is not supported`);
859
1462
  return (await git(['worktree', ...args], at, true)).code;
1463
+ }
860
1464
  path = arg;
861
1465
  break;
862
1466
  }
@@ -866,14 +1470,24 @@ export function createGitDedup(options = {}) {
866
1470
  const target = resolve(at, path);
867
1471
  // Git already shares the superproject object directory between worktrees.
868
1472
  // Its submodules have separate gitdirs, so populate those through the pool.
869
- if (await isPresent(join(target, '.gitmodules')))
870
- return updateSubmodules(['update', '--init', '--recursive'], target);
1473
+ // Like plain git worktree add, succeed once the worktree exists; submodule setup is extra.
1474
+ if ((await isPresent(join(target, '.gitmodules'))) &&
1475
+ (await updateSubmodules(['update', '--init', '--recursive', ...(quiet ? ['-q'] : [])], target)) !== 0)
1476
+ process.stderr.write(`git-dedup: worktree created, but submodule setup failed; retry with \`git submodule update --init --recursive\` in ${target}\n`);
871
1477
  return 0;
872
1478
  }
873
1479
  async function run(args) {
874
1480
  const parsed = parseGlobal(args, cwd);
875
- if (incomingEnv.GITX_ACTIVE === '1' || incomingEnv.GIT_DEDUP_ACTIVE === '1' || hasRepositoryEnvironment || !parsed)
1481
+ if (incomingEnv.GITX_ACTIVE === '1' || incomingEnv.GIT_DEDUP_ACTIVE === '1' || !parsed)
1482
+ return (await git(args, cwd, true)).code;
1483
+ const managed = parsed.command === 'clone' ||
1484
+ (parsed.command === 'submodule' && ['add', 'update'].includes(parsed.rest[0])) ||
1485
+ (parsed.command === 'worktree' && parsed.rest[0] === 'add');
1486
+ if (hasRepositoryEnvironment) {
1487
+ if (managed)
1488
+ fallback(parsed.rest, 'Git repository environment variables are set');
876
1489
  return (await git(args, cwd, true)).code;
1490
+ }
877
1491
  // -C is resolved explicitly. Other global options can change Git semantics, so forward intact.
878
1492
  for (let i = 0; i < parsed.prefix.length; i++) {
879
1493
  const arg = parsed.prefix[i];
@@ -881,8 +1495,10 @@ export function createGitDedup(options = {}) {
881
1495
  i++;
882
1496
  continue;
883
1497
  }
884
- if (arg.startsWith('-C'))
885
- continue;
1498
+ if (managed)
1499
+ fallback(parsed.rest, `global option ${arg.split('=')[0]} is not supported`);
1500
+ // ponytail: a clone forwarded here (or with repository environment variables) is not checked for pool
1501
+ // borrowing; options like -C and --git-dir move where it lands. Pin it with `store add` if needed.
886
1502
  return (await git(args, cwd, true)).code;
887
1503
  }
888
1504
  if (parsed.cwd !== cwd) {
@@ -906,7 +1522,7 @@ export function createGitDedup(options = {}) {
906
1522
  throw new Error(`git --version failed: ${result.stderr.trim()}`);
907
1523
  return result.stdout.trim();
908
1524
  }
909
- return { run, cache, storeInfo, listRemotes, fetch, gc, doctor, storePath, gitVersion };
1525
+ return { run, add, remove, storeInfo, listRemotes, fetch, gc, prune, forget, doctor, storePath, gitVersion, gitPath };
910
1526
  }
911
1527
  function processExists(pid) {
912
1528
  try {