@enbox/gitd 0.0.3 → 0.2.0

package/src/cli/commands/migrate.ts

@@ -2,47 +2,136 @@
  * `gitd migrate` — import repository data from GitHub.
  *
  * Fetches repo metadata, issues, pull requests, and releases from the
- * GitHub REST API and creates corresponding DWN records.  A `GITHUB_TOKEN`
- * env var is recommended for authenticated requests (higher rate limits).
+ * GitHub REST API and creates corresponding DWN records.
+ *
+ * Authentication is resolved automatically:
+ *   1. `GITHUB_TOKEN` env var (if set)
+ *   2. `gh auth token` (GitHub CLI, if installed and authenticated)
+ *
+ * The `<owner/repo>` argument is optional — if omitted, it is detected
+ * from the `origin` (or `github`) remote of the current git repository.
  *
  * Usage:
- *   gitd migrate all <owner/repo>            Import everything
- *   gitd migrate repo <owner/repo>           Import repo metadata only
- *   gitd migrate issues <owner/repo>         Import issues + comments
- *   gitd migrate pulls <owner/repo>          Import PRs as patches + reviews
- *   gitd migrate releases <owner/repo>       Import releases
+ *   gitd migrate all [owner/repo]            Import everything
+ *   gitd migrate repo [owner/repo]           Import repo metadata only
+ *   gitd migrate issues [owner/repo]         Import issues + comments
+ *   gitd migrate pulls [owner/repo]          Import PRs as patches + reviews
+ *   gitd migrate releases [owner/repo]       Import releases
  *
  * @module
  */
 
 import type { AgentContext } from '../agent.js';
 
+import { createFullBundle } from '../../git-server/bundle-sync.js';
+import { flagValue } from '../flags.js';
+import { getRepoContext } from '../repo-context.js';
 import { getRepoContextId } from '../repo-context.js';
+import { GitBackend } from '../../git-server/git-backend.js';
+import { readGitRefs } from '../../git-server/ref-sync.js';
+import { readFile, unlink } from 'node:fs/promises';
+import { spawn, spawnSync } from 'node:child_process';
 
 // ---------------------------------------------------------------------------
-// GitHub API helpers
+// GitHub auth — resolve a token from env or the gh CLI
 // ---------------------------------------------------------------------------
 
 const GITHUB_API = 'https://api.github.com';
 
+/** Cached token so we only resolve once per process. */
+let cachedToken: string | undefined;
+
+/** Reset the cached token (for testing). */
+export function resetTokenCache(): void {
+  cachedToken = undefined;
+}
+
+/**
+ * Resolve a GitHub API token.
+ *
+ * Priority:
+ *   1. `GITHUB_TOKEN` environment variable
+ *   2. `gh auth token` (GitHub CLI)
+ *
+ * Returns `undefined` when no token is available.
+ */
+export function resolveGitHubToken(): string | undefined {
+  if (cachedToken !== undefined) { return cachedToken; }
+
+  // 1. Env var takes precedence.
+  const envToken = process.env.GITHUB_TOKEN;
+  if (envToken) {
+    cachedToken = envToken;
+    return cachedToken;
+  }
+
+  // 2. Try the GitHub CLI.
+  try {
+    const result = spawnSync('gh', ['auth', 'token'], {
+      stdio   : ['pipe', 'pipe', 'pipe'],
+      timeout : 2_000,
+    });
+    const token = result.stdout?.toString().trim();
+    if (result.status === 0 && token) {
+      cachedToken = token;
+      return cachedToken;
+    }
+  } catch {
+    // gh not installed or not on PATH — fall through.
+  }
+
+  cachedToken = ''; // empty string = resolved, nothing found
+  return undefined;
+}
+
+// ---------------------------------------------------------------------------
+// GitHub API helpers
+// ---------------------------------------------------------------------------
+
 /** Headers for GitHub API requests. */
 function githubHeaders(): Record<string, string> {
   const headers: Record<string, string> = {
     'Accept'     : 'application/vnd.github+json',
     'User-Agent' : 'gitd-migrate/0.1',
   };
-  const token = process.env.GITHUB_TOKEN;
+  const token = resolveGitHubToken();
   if (token) { headers['Authorization'] = `Bearer ${token}`; }
   return headers;
 }
 
+/**
+ * Build a user-friendly error message for GitHub API failures.
+ * For 404s without a token, hint at authentication.
+ */
+function ghErrorMessage(status: number, body: string): string {
+  const base = `GitHub API ${status}: ${body.slice(0, 200)}`;
+
+  if (status === 404 && !resolveGitHubToken()) {
+    return (
+      `${base}\n\n` +
+      `  Hint: this may be a private repo.  Authenticate with one of:\n` +
+      `    - gh auth login          (GitHub CLI — recommended)\n` +
+      `    - export GITHUB_TOKEN=ghp_...\n`
+    );
+  }
+
+  if (status === 401 || status === 403) {
+    return (
+      `${base}\n\n` +
+      `  Hint: your token may lack the required scopes.  Ensure it has "repo" access.\n`
+    );
+  }
+
+  return base;
+}
+
 /** Fetch a single page from the GitHub API.  Throws on non-2xx. */
 async function ghFetch<T>(path: string): Promise<T> {
   const url = `${GITHUB_API}${path}`;
   const res = await fetch(url, { headers: githubHeaders() });
   if (!res.ok) {
     const body = await res.text();
-    throw new Error(`GitHub API ${res.status}: ${body.slice(0, 200)}`);
+    throw new Error(ghErrorMessage(res.status, body));
   }
   return res.json() as Promise<T>;
 }
@@ -59,7 +148,7 @@ async function ghFetchAll<T>(path: string, perPage = 100): Promise<T[]> {
     const res: Response = await fetch(url, { headers: githubHeaders() });
     if (!res.ok) {
       const body = await res.text();
-      throw new Error(`GitHub API ${res.status}: ${body.slice(0, 200)}`);
+      throw new Error(ghErrorMessage(res.status, body));
     }
 
     const items = await res.json() as T[];
@@ -147,31 +236,88 @@ export async function migrateCommand(ctx: AgentContext, args: string[]): Promise
     case 'pulls': return migratePulls(ctx, rest);
     case 'releases': return migrateReleases(ctx, rest);
     default:
-      console.error('Usage: gitd migrate <all|repo|issues|pulls|releases> <owner/repo>');
+      console.error('Usage: gitd migrate <all|repo|issues|pulls|releases> [owner/repo] [--repos <path>]');
       process.exit(1);
   }
 }
 
 // ---------------------------------------------------------------------------
-// Parse <owner/repo> argument
+// Resolve <owner/repo> — from argument or local git remote
 // ---------------------------------------------------------------------------
 
-function parseGhRepo(args: string[]): { owner: string; repo: string } {
-  const target = args[0];
-  if (!target || !target.includes('/')) {
-    console.error('Usage: gitd migrate <subcommand> <owner/repo>');
-    process.exit(1);
+/** Well-known GitHub remote URL patterns. */
+const GH_SSH_RE = /^git@github\.com:(.+?)\/(.+?)(?:\.git)?$/;
+const GH_HTTPS_RE = /^https?:\/\/github\.com\/(.+?)\/(.+?)(?:\.git)?$/;
+
+/**
+ * Extract `owner/repo` from a GitHub remote URL.
+ * Supports both SSH (`git@github.com:owner/repo.git`) and
+ * HTTPS (`https://github.com/owner/repo.git`) forms.
+ */
+export function parseGitHubRemote(url: string): { owner: string; repo: string } | null {
+  const ssh = GH_SSH_RE.exec(url);
+  if (ssh) { return { owner: ssh[1], repo: ssh[2] }; }
+
+  const https = GH_HTTPS_RE.exec(url);
+  if (https) { return { owner: https[1], repo: https[2] }; }
+
+  return null;
+}
+
+/**
+ * Detect the GitHub `owner/repo` from the current directory's git remotes.
+ * Checks `origin` first, then `github`.
+ */
+function detectGhRepoFromRemotes(): { owner: string; repo: string } | null {
+  for (const remoteName of ['origin', 'github']) {
+    try {
+      const result = spawnSync('git', ['remote', 'get-url', remoteName], {
+        stdio   : ['pipe', 'pipe', 'pipe'],
+        timeout : 5_000,
+      });
+      const url = result.stdout?.toString().trim();
+      if (result.status === 0 && url) {
+        const parsed = parseGitHubRemote(url);
+        if (parsed) { return parsed; }
+      }
+    } catch {
+      // git not available or not a repo — continue.
+    }
   }
+  return null;
+}
 
-  const [owner, ...repoParts] = target.split('/');
-  const repo = repoParts.join('/');
+/**
+ * Resolve the GitHub `owner/repo` to migrate.
+ *
+ * 1. If `args[0]` contains a `/`, treat it as an explicit `owner/repo`.
+ * 2. Otherwise, detect from the current directory's git remotes.
+ * 3. If neither works, print an error and exit.
+ */
+export function resolveGhRepo(args: string[]): { owner: string; repo: string } {
+  const target = args[0];
 
-  if (!owner || !repo) {
-    console.error('Invalid repository format. Use: owner/repo');
-    process.exit(1);
+  // Explicit argument.
+  if (target && target.includes('/')) {
+    const [owner, ...repoParts] = target.split('/');
+    const repo = repoParts.join('/');
+    if (owner && repo) { return { owner, repo }; }
+  }
+
+  // Auto-detect from git remotes.
+  const detected = detectGhRepoFromRemotes();
+  if (detected) {
+    console.log(`Detected GitHub repo: ${detected.owner}/${detected.repo}`);
+    return detected;
   }
 
-  return { owner, repo };
+  console.error(
+    'Could not determine GitHub repository.\n' +
+    '  Either pass owner/repo explicitly:\n' +
+    '    gitd migrate <subcommand> <owner/repo>\n' +
+    '  Or run from inside a git repo with a GitHub remote.\n',
+  );
+  process.exit(1);
 }
 
 /**
@@ -187,23 +333,52 @@ function prependAuthor(body: string, ghLogin: string): string {
 // migrate all
 // ---------------------------------------------------------------------------
 
+/**
+ * Resolve the repos base path from CLI flags or environment.
+ * Returns `null` when no explicit path was provided, so callers can
+ * decide whether to skip git content migration.
+ */
+function resolveReposPath(args: string[]): string | null {
+  return flagValue(args, '--repos') ?? process.env.GITD_REPOS ?? null;
+}
+
 async function migrateAll(ctx: AgentContext, args: string[]): Promise<void> {
-  const { owner, repo } = parseGhRepo(args);
+  const { owner, repo } = resolveGhRepo(args);
+  const reposPath = resolveReposPath(args);
   const slug = `${owner}/${repo}`;
 
+  const token = resolveGitHubToken();
+  if (!token) {
+    console.log(`Warning: no GitHub token found. Private repos will fail.`);
+    console.log(`  Run "gh auth login" or set GITHUB_TOKEN to authenticate.\n`);
+  }
+
   console.log(`Migrating ${slug} from GitHub...\n`);
 
   try {
     // Step 1: repo metadata.
     await migrateRepoInner(ctx, owner, repo);
 
-    // Step 2: issues + comments.
+    // Step 2: git content (clone, bundle, refs).
+    if (reposPath) {
+      try {
+        await migrateGitContent(ctx, owner, repo, reposPath);
+      } catch (err) {
+        console.error(`  Warning: git content migration failed: ${(err as Error).message}`);
+        console.error('  Metadata, issues, PRs, and releases will still be imported.');
+        console.error('  Re-run with a valid --repos path to retry git content migration.\n');
+      }
+    } else {
+      console.log('  Skipping git content — pass --repos <path> or set GITD_REPOS to include git data.');
+    }
+
+    // Step 3: issues + comments.
     const issueCount = await migrateIssuesInner(ctx, owner, repo);
 
-    // Step 3: pull requests + reviews.
+    // Step 4: pull requests + reviews.
     const pullCount = await migratePullsInner(ctx, owner, repo);
 
-    // Step 4: releases.
+    // Step 5: releases.
     const releaseCount = await migrateReleasesInner(ctx, owner, repo);
 
     console.log(`\nMigration complete: ${slug}`);
@@ -221,9 +396,15 @@ async function migrateAll(ctx: AgentContext, args: string[]): Promise<void> {
 // ---------------------------------------------------------------------------
 
 async function migrateRepo(ctx: AgentContext, args: string[]): Promise<void> {
-  const { owner, repo } = parseGhRepo(args);
+  const { owner, repo } = resolveGhRepo(args);
+  const reposPath = resolveReposPath(args);
   try {
     await migrateRepoInner(ctx, owner, repo);
+    if (reposPath) {
+      await migrateGitContent(ctx, owner, repo, reposPath);
+    } else {
+      console.log('  Skipping git content — pass --repos <path> or set GITD_REPOS to include git data.');
+    }
   } catch (err) {
     console.error(`Failed to migrate repo: ${(err as Error).message}`);
     process.exit(1);
@@ -267,12 +448,137 @@ async function migrateRepoInner(ctx: AgentContext, owner: string, repo: string):
   console.log(`  Source:    ${gh.html_url}`);
 }
 
+// ---------------------------------------------------------------------------
+// migrate git content (clone + bundle + refs)
+// ---------------------------------------------------------------------------
+
+/**
+ * Clone the GitHub repository as a bare repo, create a full git bundle,
+ * upload it to DWN, and sync all refs to DWN records.
+ *
+ * This is the "git content" migration step that turns the metadata-only
+ * repo record into a fully cloneable repo.
+ */
+async function migrateGitContent(
+  ctx: AgentContext,
+  owner: string,
+  repo: string,
+  reposPath: string,
+): Promise<void> {
+  const slug = `${owner}/${repo}`;
+  console.log(`Importing git content from ${slug}...`);
+
+  const backend = new GitBackend({ basePath: reposPath });
+  const repoPath = backend.repoPath(ctx.did, repo);
+
+  // Step 1: Clone bare repo from GitHub (or skip if already on disk).
+  if (backend.exists(ctx.did, repo)) {
+    console.log(`  Bare repo already exists at ${repoPath} — skipping clone.`);
+  } else {
+    const cloneUrl = buildCloneUrl(owner, repo);
+    console.log(`  Cloning ${cloneUrl} → ${repoPath}`);
+    await cloneBare(cloneUrl, repoPath);
+    console.log('  Clone complete.');
+  }
+
+  // Step 2: Create full git bundle.
+  console.log('  Creating git bundle...');
+  const bundleInfo = await createFullBundle(repoPath);
+  console.log(`  Bundle: ${bundleInfo.size} bytes, ${bundleInfo.refCount} ref(s), tip: ${bundleInfo.tipCommit.slice(0, 8)}`);
+
+  // Step 3: Upload bundle to DWN.
+  const { contextId: repoContextId, visibility } = await getRepoContext(ctx);
+  const encrypt = visibility === 'private';
+
+  try {
+    const bundleData = new Uint8Array(await readFile(bundleInfo.path));
+
+    const { status } = await ctx.repo.records.create('repo/bundle', {
+      data       : bundleData,
+      dataFormat : 'application/x-git-bundle',
+      tags       : {
+        tipCommit : bundleInfo.tipCommit,
+        isFull    : true,
+        refCount  : bundleInfo.refCount,
+        size      : bundleInfo.size,
+      },
+      parentContextId : repoContextId,
+      encryption      : encrypt,
+    } as any);
+
+    if (status.code >= 300) {
+      throw new Error(`Failed to create bundle record: ${status.code} ${status.detail}`);
+    }
+    console.log('  Bundle uploaded to DWN.');
+  } finally {
+    await unlink(bundleInfo.path).catch(() => {});
+  }
+
+  // Step 4: Sync git refs to DWN.
+  console.log('  Syncing refs to DWN...');
+  const gitRefs = await readGitRefs(repoPath);
+
+  let refCount = 0;
+  for (const ref of gitRefs) {
+    const { status } = await ctx.refs.records.create('repo/ref', {
+      data            : { name: ref.name, target: ref.target, type: ref.type },
+      tags            : { name: ref.name, type: ref.type, target: ref.target },
+      parentContextId : repoContextId,
+    });
+
+    if (status.code >= 300) {
+      console.error(`  Failed to sync ref ${ref.name}: ${status.code} ${status.detail}`);
+      continue;
+    }
+    refCount++;
+  }
+
+  console.log(`  Synced ${refCount} ref(s) to DWN.`);
+  console.log(`  Git content migration complete.`);
+}
+
+/**
+ * Build the clone URL for a GitHub repository.
+ * Uses the token (if available) for authenticated HTTPS clone.
+ */
+function buildCloneUrl(owner: string, repo: string): string {
+  const token = resolveGitHubToken();
+  if (token) {
+    return `https://${token}@github.com/${owner}/${repo}.git`;
+  }
+  return `https://github.com/${owner}/${repo}.git`;
+}
+
+/**
+ * Clone a git repository as a bare repo using `git clone --bare`.
+ */
+function cloneBare(url: string, destPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn('git', ['clone', '--bare', url, destPath], {
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    const stderrChunks: Buffer[] = [];
+    child.stderr!.on('data', (chunk: Buffer) => stderrChunks.push(chunk));
+
+    child.on('error', reject);
+    child.on('exit', (code: number | null) => {
+      if (code !== 0) {
+        const stderr = Buffer.concat(stderrChunks).toString('utf-8');
+        reject(new Error(`git clone --bare failed (exit ${code}): ${stderr}`));
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
 // ---------------------------------------------------------------------------
 // migrate issues
 // ---------------------------------------------------------------------------
 
 async function migrateIssues(ctx: AgentContext, args: string[]): Promise<void> {
-  const { owner, repo } = parseGhRepo(args);
+  const { owner, repo } = resolveGhRepo(args);
   try {
     const count = await migrateIssuesInner(ctx, owner, repo);
     console.log(`\nImported ${count} issue${count !== 1 ? 's' : ''}.`);
@@ -356,7 +662,7 @@ async function migrateIssuesInner(ctx: AgentContext, owner: string, repo: string
 // ---------------------------------------------------------------------------
 
 async function migratePulls(ctx: AgentContext, args: string[]): Promise<void> {
-  const { owner, repo } = parseGhRepo(args);
+  const { owner, repo } = resolveGhRepo(args);
   try {
     const count = await migratePullsInner(ctx, owner, repo);
     console.log(`\nImported ${count} patch${count !== 1 ? 'es' : ''}.`);
@@ -467,7 +773,7 @@ async function migratePullsInner(ctx: AgentContext, owner: string, repo: string)
 // ---------------------------------------------------------------------------
 
 async function migrateReleases(ctx: AgentContext, args: string[]): Promise<void> {
-  const { owner, repo } = parseGhRepo(args);
+  const { owner, repo } = resolveGhRepo(args);
   try {
     const count = await migrateReleasesInner(ctx, owner, repo);
     console.log(`\nImported ${count} release${count !== 1 ? 's' : ''}.`);

package/src/cli/main.ts

@@ -43,10 +43,10 @@
  *   gitd social star <did>                  Star a repo
  *   gitd social follow <did>                Follow a user
  *   gitd notification list [--unread]       List notifications
- *   gitd migrate all <owner/repo>            Import everything from GitHub
- *   gitd migrate issues <owner/repo>         Import issues + comments
- *   gitd migrate pulls <owner/repo>          Import PRs as patches
- *   gitd migrate releases <owner/repo>       Import releases
+ *   gitd migrate all [owner/repo]             Import everything from GitHub
+ *   gitd migrate issues [owner/repo]          Import issues + comments
+ *   gitd migrate pulls [owner/repo]           Import PRs as patches
+ *   gitd migrate releases [owner/repo]        Import releases
  *   gitd web [--port <port>]                Start the read-only web UI
  *   gitd indexer [--port] [--interval] [--seed]  Start the indexer service
  *   gitd daemon [--config <path>] [--only ...] Start unified shim daemon
@@ -69,6 +69,7 @@
 import { ciCommand } from './commands/ci.js';
 import { cloneCommand } from './commands/clone.js';
 import { connectAgent } from './agent.js';
+import { createRequire } from 'node:module';
 import { daemonCommand } from './commands/daemon.js';
 import { githubApiCommand } from './commands/github-api.js';
 import { indexerCommand } from '../indexer/main.js';
@@ -170,11 +171,11 @@ function printUsage(): void {
   console.log('  notification read <id>                      Mark as read');
   console.log('  notification clear                          Clear read notifications');
   console.log('');
-  console.log('  migrate all <owner/repo>                   Import everything from GitHub');
-  console.log('  migrate repo <owner/repo>                  Import repo metadata');
-  console.log('  migrate issues <owner/repo>                Import issues + comments');
-  console.log('  migrate pulls <owner/repo>                 Import PRs as patches + reviews');
-  console.log('  migrate releases <owner/repo>              Import releases');
+  console.log('  migrate all [owner/repo]                    Import everything from GitHub');
+  console.log('  migrate repo [owner/repo]                   Import repo metadata');
+  console.log('  migrate issues [owner/repo]                 Import issues + comments');
+  console.log('  migrate pulls [owner/repo]                  Import PRs as patches + reviews');
+  console.log('  migrate releases [owner/repo]               Import releases');
   console.log('');
   console.log('  web [--port <port>]                         Start read-only web UI (default: 8080)');
   console.log('');
@@ -204,7 +205,7 @@ function printUsage(): void {
   console.log('  GITD_NPM_SHIM_PORT    npm shim port (default: 4873)');
   console.log('  GITD_GO_SHIM_PORT     Go proxy shim port (default: 4874)');
   console.log('  GITD_OCI_SHIM_PORT    OCI registry shim port (default: 5555)');
-  console.log('  GITHUB_TOKEN      GitHub API token for migration (optional, higher rate limits)');
+  console.log('  GITHUB_TOKEN      GitHub API token for migration (auto-detected from gh CLI)');
 }
 
 // ---------------------------------------------------------------------------
@@ -216,8 +217,49 @@ async function getPassword(): Promise<string> {
   const env = process.env.GITD_PASSWORD;
   if (env) { return env; }
 
-  // Interactive prompt via stdin.
+  // Interactive prompt — hide input when running in a TTY.
   process.stdout.write('Vault password: ');
+
+  if (process.stdin.isTTY) {
+    // Raw mode: read character-by-character, echo nothing.
+    const password = await new Promise<string>((resolve) => {
+      let buf = '';
+      process.stdin.setRawMode(true);
+      process.stdin.setEncoding('utf8');
+      process.stdin.resume();
+
+      const onData = (ch: string): void => {
+        const code = ch.charCodeAt(0);
+
+        if (ch === '\r' || ch === '\n') {
+          // Enter — done.
+          process.stdin.setRawMode(false);
+          process.stdin.pause();
+          process.stdin.removeListener('data', onData);
+          process.stdout.write('\n');
+          resolve(buf);
+        } else if (code === 3) {
+          // Ctrl-C — abort.
+          process.stdin.setRawMode(false);
+          process.stdout.write('\n');
+          process.exit(130);
+        } else if (code === 127 || code === 8) {
+          // Backspace / Delete.
+          if (buf.length > 0) {
+            buf = buf.slice(0, -1);
+          }
+        } else if (code >= 32) {
+          // Printable character.
+          buf += ch;
+        }
+      };
+
+      process.stdin.on('data', onData);
+    });
+    return password;
+  }
+
+  // Non-TTY fallback (piped input).
   const response = await new Promise<string>((resolve) => {
     let buf = '';
     process.stdin.setEncoding('utf8');
@@ -234,7 +276,18 @@ async function getPassword(): Promise<string> {
 // Main
 // ---------------------------------------------------------------------------
 
+function printVersion(): void {
+  const require = createRequire(import.meta.url);
+  const pkg = require('../../package.json') as { version: string };
+  console.log(`gitd ${pkg.version}`);
+}
+
 async function main(): Promise<void> {
+  if (command === '--version' || command === '-v' || command === 'version') {
+    printVersion();
+    return;
+  }
+
   if (!command || command === 'help' || command === '--help' || command === '-h') {
     printUsage();
     return;
@@ -342,6 +395,13 @@ async function main(): Promise<void> {
       printUsage();
       process.exit(1);
   }
+
+  // One-shot commands reach here after completing.  The Web5 agent keeps
+  // LevelDB stores and other handles open, which prevents the process from
+  // exiting naturally.  Long-running commands (serve, web, daemon, indexer,
+  // github-api, shim) never reach this point because they block on an
+  // infinite promise internally.
+  process.exit(0);
 }
 
 main().catch((err: Error) => {