gdkbox 0.1.10 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75c5c8ed1f89cb877856f68e45c39f4b5460131ea5a312e5af10e7cb7de2688b
-  data.tar.gz: a0eb3697bb1b630665be5799d33ffb8bb4aec0999858e075861e3469bd312431
+  metadata.gz: 8df6faa5d2c2a7a0fe888195af92305774354b997ef5e0e2d1a7ee61469b60bf
+  data.tar.gz: 4c859a064721f68cdadc7a7140dc73b0d3a179a57fe3b538c63699c021115109
 SHA512:
-  metadata.gz: 344bba38a105dcf4bf899893d18eb7ef9f82ef03ef6d2229af810346e27f18b24c46e09c02c7f83e406255f79e8d033ff37c159ced868aad284e540c6cf7948c
-  data.tar.gz: ce33e8b7bd1aa403ea5fe77127b7c46fd1c4eb7079736689cc6447b6b0649bd270b880db82d31a33bb0d090f818366b1e2de847e6b3d70f918dbae257200014a
+  metadata.gz: 9ebd61583a5e01145206ba5407e2fbaad1cf456bbfbb040ecc58e66d0d471c3a5566fca4d194467f9dcc988f5540f57f54c73edf96ac5805248a3c34996f4095
+  data.tar.gz: fd01809df5f90d1eff5229918f424c311577f15a756ea32328157ebdcaf237e5d1c998a1db87d004a07dbc77be413c4260cc17baaa9db58248f5087cd3488467

data/README.md

@@ -92,6 +92,7 @@ Or run it straight from the checkout without installing:
 | `gdkbox set-host` | Add the `gdk.local` entry to your `/etc/hosts` so box web UIs resolve (`--remove` to undo). Also runs during `up`. |
 | `gdkbox set-git NAME` | Seed your git identity (`user.name`/`user.email`) into the box so `git commit` works. Also runs during `up`. |
 | `gdkbox set-remote NAME [REMOTE]` | Point the box's GitLab checkout at a different remote (URL or `namespace/project`, e.g. `gitlab-org/gitlab`); default from config.yml. |
+| `gdkbox hydrate NAME` | Backfill the box's treeless GitLab clone so deep rebases/blame/bisect work (`--trees` for a much smaller trees-only fetch). |
 | `gdkbox start NAME` | Start a stopped box (and re-enable SSH). |
 | `gdkbox stop NAME` | Stop a running box. |
 | `gdkbox rm NAME` | Remove a box: container, metadata, and SSH entry. |
@@ -136,6 +137,23 @@ gdkbox stop demo
 gdkbox rm demo
 ```
 
+### Working with git history in a box
+
+The image's GitLab checkout is a **treeless clone** (`--filter=tree:0`, ~550 MB
+instead of many GB). Day-to-day work near `master` is fine, but operations that
+span a large history gap — rebasing a branch hundreds of commits behind,
+`git blame` or `git bisect` across old ranges — trigger storms of lazy object
+fetches and fail with errors like `could not fetch <sha> from promisor remote`
+(not merge conflicts, even when a rebase reports failure). Either keep branches
+current and do deep-history work elsewhere, or hydrate the box once:
+
+```sh
+gdkbox hydrate demo --trees   # fetch missing trees only: much smaller, usually
+                              # enough for deep rebases (blobs stay lazy)
+gdkbox hydrate demo           # full backfill: several GB, then the clone
+                              # behaves like a normal full clone
+```
+
 ## Orchestrating a fleet of agents
 
 The end goal of `gdkbox` is to back an **orchestrator** that runs a pool of

data/lib/gdkbox/box.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "shellwords"
 require "time"
 
 module GDKBox
@@ -184,15 +185,85 @@ module GDKBox
       "git@gitlab.com:#{token}.git"
     end
 
+    # The host an SSH-style git URL connects to, or nil for non-SSH remotes
+    # (https needs no SSH client preparation).
+    def self.remote_ssh_host(url)
+      if (m = url.match(%r{\Assh://(?:[^@/]+@)?([^:/]+)}))
+        m[1]
+      elsif (m = url.match(/\A(?:[^@:\/]+@)([^:]+):/))
+        m[1]
+      end
+    end
+
     # Point the box's GitLab checkout at a different remote (URL or
-    # "namespace/project"). Returns the URL that origin now uses.
+    # "namespace/project"). SSH remotes get the box's SSH client prepared
+    # first (host key acceptance + connection multiplexing) so the follow-up
+    # fetch — and later lazy fetches from the treeless clone — succeed.
+    # Returns the URL that origin now uses.
     def set_gitlab_remote!(remote)
       url = self.class.expand_remote(remote)
-      Provisioner.new(docker: @docker, config: @config)
-        .setup_gitlab_remote(container_name, url)
+      provisioner = Provisioner.new(docker: @docker, config: @config)
+      if (host = self.class.remote_ssh_host(url))
+        provisioner.setup_git_ssh(container_name, host)
+      end
+      provisioner.setup_gitlab_remote(container_name, url)
       url
     end
 
+    # The hydrate fetch, run inside the box. The filter is passed to `git
+    # fetch` directly and only persisted to config after the fetch succeeds
+    # (and, for a full hydrate, the previous filter is put back on failure),
+    # so an interrupted hydrate cannot leave the config claiming objects the
+    # store does not have (issue #8).
+    HYDRATE_SCRIPT = <<~'BASH'
+      set -e
+      if [ -n "$GDKBOX_HYDRATE_FILTER" ]; then
+        git fetch --refetch --progress --filter="$GDKBOX_HYDRATE_FILTER" origin
+        git config remote.origin.partialclonefilter "$GDKBOX_HYDRATE_FILTER"
+      else
+        prev=$(git config --get remote.origin.partialclonefilter || true)
+        git config --unset-all remote.origin.partialclonefilter || true
+        if ! git fetch --refetch --progress origin; then
+          if [ -n "$prev" ]; then git config remote.origin.partialclonefilter "$prev"; fi
+          echo "hydrate: fetch failed; previous filter config restored" >&2
+          exit 1
+        fi
+        git config --unset remote.origin.promisor || true
+      fi
+    BASH
+
+    # Backfill the box's treeless GitLab clone so deep-history operations
+    # (old rebases, blame, bisect) work. trees: true fetches only the missing
+    # trees (much smaller; file contents stay lazy).
+    #
+    # The fetch runs over SSH (`ssh -F <generated config> <alias>`), not
+    # `docker exec`: the remote is typically git@gitlab.com, and only an SSH
+    # session carries the user's forwarded agent — docker exec has no
+    # SSH_AUTH_SOCK, so it cannot authenticate at all (issue #8). The SSH
+    # client stanza (host key + multiplexing) is seeded first for SSH
+    # origins. Streams git's progress lines to the block; raises
+    # CommandError when the fetch fails.
+    def hydrate!(trees: false, &progress)
+      origin = @docker.exec(
+        container_name, "git remote get-url origin",
+        user: @config.ssh_user, workdir: @config.gitlab_checkout_path
+      ).stdout.strip
+      if (host = self.class.remote_ssh_host(origin))
+        Provisioner.new(docker: @docker, config: @config)
+          .setup_git_ssh(container_name, host)
+      end
+
+      filter = trees ? "blob:none" : ""
+      remote_cmd = "cd #{Shellwords.escape(@config.gitlab_checkout_path)} && " \
+        "GDKBOX_HYDRATE_FILTER=#{Shellwords.escape(filter)} " \
+        "bash -c #{Shellwords.escape(HYDRATE_SCRIPT)}"
+      argv = ["ssh", "-F", @config.ssh_config_path, ssh_host_alias, remote_cmd]
+      result = @shell.stream_tty(*argv, &progress)
+      raise CommandError.new(argv, result.status, "") unless result.success?
+
+      result
+    end
+
     # Seed a git identity into the box so `git commit` works there. Falls back
     # to the host's `git config` when name/email are not given; raises when
     # neither source has anything to seed. Returns the [name, email] seeded.

data/lib/gdkbox/cli.rb

@@ -288,6 +288,42 @@ module GDKBox
     end
     map "set-remote" => :set_remote
 
+    desc "hydrate NAME", "Backfill the box's treeless GitLab clone (deep rebases, blame, bisect)"
+    long_desc <<~DESC
+      The GDK-in-a-box image clones GitLab treeless (--filter=tree:0), so
+      operations spanning a large history gap — rebasing a branch far behind
+      master, blame or bisect across old ranges — storm the network with lazy
+      fetches and fail. Hydrating backfills the missing objects once.
+
+      By default everything is fetched and the clone becomes a normal full
+      clone (a large one-time download, likely several GB). With --trees only
+      the missing trees are fetched — a fraction of the size, and usually
+      enough for deep rebases; file contents are still fetched lazily as
+      needed. Progress is streamed live.
+    DESC
+    option :trees, type: :boolean, default: false,
+      desc: "Fetch missing trees only (much smaller; blobs stay lazy)"
+    option :force, type: :boolean, default: false, desc: "Skip confirmation"
+    def hydrate(name)
+      box = load_box!(name)
+      unless options[:force]
+        estimate = options[:trees] ? "a sizeable download" : "a large download, likely several GB"
+        return unless yes?("Hydrate '#{name}' (#{estimate})? [y/N]")
+      end
+
+      say "Hydrating '#{name}' (#{options[:trees] ? 'trees only' : 'full'})...", :green
+      tty = $stderr.tty?
+      box.hydrate!(trees: options[:trees]) do |line|
+        if tty
+          $stderr.print("\r\e[K#{line}")
+        elsif !line.include?("%") # skip per-percent spam when piped
+          $stderr.puts(line)
+        end
+      end
+      $stderr.print("\n") if tty
+      say "Done. '#{name}' can now work across the full history.", :green
+    end
+
     desc "set-host", "Add (default) or remove the gdk.local entry in /etc/hosts"
     long_desc <<~DESC
       GDK generates URLs and redirects that use the `gdk.local` hostname, so
@@ -475,10 +511,11 @@ module GDKBox
       url = box.set_gitlab_remote!(remote)
       say "Pointed the GitLab checkout at #{url}.", :green unless options[:json]
     rescue StandardError => e
-      unless options[:json]
-        say "Could not repoint the GitLab checkout: #{e.message}", :yellow
-        say "  Retry later with: gdkbox set-remote #{box.name} #{remote}", :yellow
-      end
+      # Warn on stderr even in --json mode: stdout stays clean JSON, but a
+      # silently failed repoint costs the user a debugging session later
+      # (the box looks fine until the first fetch).
+      warn "gdkbox: could not repoint the GitLab checkout: #{e.message}"
+      warn "gdkbox: retry later with: gdkbox set-remote #{box.name} #{remote}"
     end
 
     # Ensure gdk.local resolves on the host so the box's web UI is reachable

data/lib/gdkbox/completion.rb

@@ -11,8 +11,8 @@ module GDKBox
     SHELLS = %w[bash zsh].freeze
 
     # Subcommands whose first positional argument is an existing box.
-    BOX_COMMANDS = %w[status dispatch ssh code install-agent set-key set-git set-remote start
-      stop rm add-skill].freeze
+    BOX_COMMANDS = %w[status dispatch ssh code install-agent set-key set-git set-remote hydrate
+      start stop rm add-skill].freeze
 
     def initialize(cli_class = CLI)
       @cli = cli_class

data/lib/gdkbox/provisioner.rb

@@ -141,6 +141,37 @@ module GDKBox
       true
     BASH
 
+    # Prepares the box's SSH client for real git traffic against a remote
+    # host (issue #6, hit when a gitlab_remote is configured):
+    #
+    # - `StrictHostKeyChecking accept-new`: a fresh box has an empty
+    #   known_hosts, so the first fetch would die on host key verification.
+    # - Connection multiplexing: lazy fetches from the treeless clone open
+    #   SSH connections in bursts; gitlab.com throttles that, which surfaces
+    #   as a misleading "Permission denied (publickey)". One shared control
+    #   connection absorbs the burst.
+    #
+    # The stanza is tagged per host, so re-runs are no-ops and different
+    # remotes each get their own block.
+    GIT_SSH_SETUP = <<~'BASH'
+      set -e
+      mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
+      config="$HOME/.ssh/config"
+      touch "$config" && chmod 600 "$config"
+      if ! grep -qF "gdkbox:git-ssh $GDKBOX_GIT_HOST" "$config"; then
+        cat >> "$config" <<EOF
+
+      # gdkbox:git-ssh $GDKBOX_GIT_HOST
+      Host $GDKBOX_GIT_HOST
+        StrictHostKeyChecking accept-new
+        ControlMaster auto
+        ControlPath ~/.ssh/cm-%r@%h-%p
+        ControlPersist 20m
+        ServerAliveInterval 30
+      EOF
+      fi
+    BASH
+
     # Points the GitLab checkout's origin at a different repository. The image
     # hardcodes the community mirror (gitlab-community/gitlab-org/gitlab);
     # team members typically want the canonical gitlab-org/gitlab. The mirror
@@ -200,6 +231,14 @@ module GDKBox
       )
     end
 
+    def setup_git_ssh(container_name, host)
+      @docker.exec(
+        container_name, GIT_SSH_SETUP,
+        user: @config.ssh_user,
+        env: { "GDKBOX_GIT_HOST" => host }
+      )
+    end
+
     def setup_gitlab_remote(container_name, url)
       @docker.exec(
         container_name, GITLAB_REMOTE_SETUP,

data/lib/gdkbox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GDKBox
-  VERSION = "0.1.10"
+  VERSION = "0.1.13"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gdkbox
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.13
 platform: ruby
 authors:
 - jotolo