gdkbox 0.1.10 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75c5c8ed1f89cb877856f68e45c39f4b5460131ea5a312e5af10e7cb7de2688b
-  data.tar.gz: a0eb3697bb1b630665be5799d33ffb8bb4aec0999858e075861e3469bd312431
+  metadata.gz: b71b4f7e72836332b32feca0a6e3e0c485f8255fcfb0f1dc93c7792c3f6fc5ab
+  data.tar.gz: 66e0331f211fd1df2b67694e42e65d8971502615b42c61093f86a3a465e06425
 SHA512:
-  metadata.gz: 344bba38a105dcf4bf899893d18eb7ef9f82ef03ef6d2229af810346e27f18b24c46e09c02c7f83e406255f79e8d033ff37c159ced868aad284e540c6cf7948c
-  data.tar.gz: ce33e8b7bd1aa403ea5fe77127b7c46fd1c4eb7079736689cc6447b6b0649bd270b880db82d31a33bb0d090f818366b1e2de847e6b3d70f918dbae257200014a
+  metadata.gz: b70d9d914b6ce31f0a644cae9260a98adb3540b0d76555a1ed4e72fc423b9602deb84b338b1b91cf29bf455546b248e875dbfa2e7fdc5dafc9b37322cb2d58be
+  data.tar.gz: 73ebbfbfb11732b3d5fb9f40ea0d795a846c61ccb24d513c51684542424d2d78a7eef919e03d5d2007f64df3c3f0704cb75f07dbc3e7b494f90f9ae892daec43

data/README.md

@@ -92,6 +92,7 @@ Or run it straight from the checkout without installing:
 | `gdkbox set-host` | Add the `gdk.local` entry to your `/etc/hosts` so box web UIs resolve (`--remove` to undo). Also runs during `up`. |
 | `gdkbox set-git NAME` | Seed your git identity (`user.name`/`user.email`) into the box so `git commit` works. Also runs during `up`. |
 | `gdkbox set-remote NAME [REMOTE]` | Point the box's GitLab checkout at a different remote (URL or `namespace/project`, e.g. `gitlab-org/gitlab`); default from config.yml. |
+| `gdkbox hydrate NAME` | Backfill the box's treeless GitLab clone so deep rebases/blame/bisect work (`--trees` for a much smaller trees-only fetch). |
 | `gdkbox start NAME` | Start a stopped box (and re-enable SSH). |
 | `gdkbox stop NAME` | Stop a running box. |
 | `gdkbox rm NAME` | Remove a box: container, metadata, and SSH entry. |
@@ -136,6 +137,23 @@ gdkbox stop demo
 gdkbox rm demo
 ```
 
+### Working with git history in a box
+
+The image's GitLab checkout is a **treeless clone** (`--filter=tree:0`, ~550 MB
+instead of many GB). Day-to-day work near `master` is fine, but operations that
+span a large history gap — rebasing a branch hundreds of commits behind,
+`git blame` or `git bisect` across old ranges — trigger storms of lazy object
+fetches and fail with errors like `could not fetch <sha> from promisor remote`
+(not merge conflicts, even when a rebase reports failure). Either keep branches
+current and do deep-history work elsewhere, or hydrate the box once:
+
+```sh
+gdkbox hydrate demo --trees   # fetch missing trees only: much smaller, usually
+                              # enough for deep rebases (blobs stay lazy)
+gdkbox hydrate demo           # full backfill: several GB, then the clone
+                              # behaves like a normal full clone
+```
+
 ## Orchestrating a fleet of agents
 
 The end goal of `gdkbox` is to back an **orchestrator** that runs a pool of

data/lib/gdkbox/box.rb

@@ -184,15 +184,49 @@ module GDKBox
       "git@gitlab.com:#{token}.git"
     end
 
+    # The host an SSH-style git URL connects to, or nil for non-SSH remotes
+    # (https needs no SSH client preparation).
+    def self.remote_ssh_host(url)
+      if (m = url.match(%r{\Assh://(?:[^@/]+@)?([^:/]+)}))
+        m[1]
+      elsif (m = url.match(/\A(?:[^@:\/]+@)([^:]+):/))
+        m[1]
+      end
+    end
+
     # Point the box's GitLab checkout at a different remote (URL or
-    # "namespace/project"). Returns the URL that origin now uses.
+    # "namespace/project"). SSH remotes get the box's SSH client prepared
+    # first (host key acceptance + connection multiplexing) so the follow-up
+    # fetch — and later lazy fetches from the treeless clone — succeed.
+    # Returns the URL that origin now uses.
     def set_gitlab_remote!(remote)
       url = self.class.expand_remote(remote)
-      Provisioner.new(docker: @docker, config: @config)
-        .setup_gitlab_remote(container_name, url)
+      provisioner = Provisioner.new(docker: @docker, config: @config)
+      if (host = self.class.remote_ssh_host(url))
+        provisioner.setup_git_ssh(container_name, host)
+      end
+      provisioner.setup_gitlab_remote(container_name, url)
       url
     end
 
+    # Backfill the box's treeless GitLab clone so deep-history operations
+    # (old rebases, blame, bisect) work. trees: true fetches only the missing
+    # trees (much smaller; file contents stay lazy). Prepares the SSH client
+    # first when origin is an SSH remote — a refetch without multiplexing
+    # would hit the same throttling the lazy fetches do. Streams git's
+    # progress lines to the block.
+    def hydrate!(trees: false, &progress)
+      provisioner = Provisioner.new(docker: @docker, config: @config)
+      origin = @docker.exec(
+        container_name, "git remote get-url origin",
+        user: @config.ssh_user, workdir: @config.gitlab_checkout_path
+      ).stdout.strip
+      if (host = self.class.remote_ssh_host(origin))
+        provisioner.setup_git_ssh(container_name, host)
+      end
+      provisioner.hydrate(container_name, trees: trees, &progress)
+    end
+
     # Seed a git identity into the box so `git commit` works there. Falls back
     # to the host's `git config` when name/email are not given; raises when
     # neither source has anything to seed. Returns the [name, email] seeded.

data/lib/gdkbox/cli.rb

@@ -288,6 +288,42 @@ module GDKBox
     end
     map "set-remote" => :set_remote
 
+    desc "hydrate NAME", "Backfill the box's treeless GitLab clone (deep rebases, blame, bisect)"
+    long_desc <<~DESC
+      The GDK-in-a-box image clones GitLab treeless (--filter=tree:0), so
+      operations spanning a large history gap — rebasing a branch far behind
+      master, blame or bisect across old ranges — storm the network with lazy
+      fetches and fail. Hydrating backfills the missing objects once.
+
+      By default everything is fetched and the clone becomes a normal full
+      clone (a large one-time download, likely several GB). With --trees only
+      the missing trees are fetched — a fraction of the size, and usually
+      enough for deep rebases; file contents are still fetched lazily as
+      needed. Progress is streamed live.
+    DESC
+    option :trees, type: :boolean, default: false,
+      desc: "Fetch missing trees only (much smaller; blobs stay lazy)"
+    option :force, type: :boolean, default: false, desc: "Skip confirmation"
+    def hydrate(name)
+      box = load_box!(name)
+      unless options[:force]
+        estimate = options[:trees] ? "a sizeable download" : "a large download, likely several GB"
+        return unless yes?("Hydrate '#{name}' (#{estimate})? [y/N]")
+      end
+
+      say "Hydrating '#{name}' (#{options[:trees] ? 'trees only' : 'full'})...", :green
+      tty = $stderr.tty?
+      box.hydrate!(trees: options[:trees]) do |line|
+        if tty
+          $stderr.print("\r\e[K#{line}")
+        elsif !line.include?("%") # skip per-percent spam when piped
+          $stderr.puts(line)
+        end
+      end
+      $stderr.print("\n") if tty
+      say "Done. '#{name}' can now work across the full history.", :green
+    end
+
     desc "set-host", "Add (default) or remove the gdk.local entry in /etc/hosts"
     long_desc <<~DESC
       GDK generates URLs and redirects that use the `gdk.local` hostname, so
@@ -475,10 +511,11 @@ module GDKBox
       url = box.set_gitlab_remote!(remote)
       say "Pointed the GitLab checkout at #{url}.", :green unless options[:json]
     rescue StandardError => e
-      unless options[:json]
-        say "Could not repoint the GitLab checkout: #{e.message}", :yellow
-        say "  Retry later with: gdkbox set-remote #{box.name} #{remote}", :yellow
-      end
+      # Warn on stderr even in --json mode: stdout stays clean JSON, but a
+      # silently failed repoint costs the user a debugging session later
+      # (the box looks fine until the first fetch).
+      warn "gdkbox: could not repoint the GitLab checkout: #{e.message}"
+      warn "gdkbox: retry later with: gdkbox set-remote #{box.name} #{remote}"
     end
 
     # Ensure gdk.local resolves on the host so the box's web UI is reachable

data/lib/gdkbox/completion.rb

@@ -11,8 +11,8 @@ module GDKBox
     SHELLS = %w[bash zsh].freeze
 
     # Subcommands whose first positional argument is an existing box.
-    BOX_COMMANDS = %w[status dispatch ssh code install-agent set-key set-git set-remote start
-      stop rm add-skill].freeze
+    BOX_COMMANDS = %w[status dispatch ssh code install-agent set-key set-git set-remote hydrate
+      start stop rm add-skill].freeze
 
     def initialize(cli_class = CLI)
       @cli = cli_class

data/lib/gdkbox/docker.rb

@@ -223,6 +223,21 @@ module GDKBox
       check ? @shell.run!(*cmd, input: input) : @shell.run(*cmd, input: input)
     end
 
+    # Like #exec, but for long-running scripts whose output should be seen
+    # live (e.g. a multi-GB git fetch): runs under a PTY and yields each
+    # output line as it appears. Raises CommandError on a non-zero exit.
+    def exec_stream(name, script, user: nil, workdir: nil, env: {}, &block)
+      cmd = ["docker", "exec"]
+      cmd.push("-u", user) if user
+      cmd.push("-w", workdir) if workdir
+      env.each { |key, value| cmd.push("-e", "#{key}=#{value}") }
+      cmd.push(name, "bash", "-lc", script)
+      result = @shell.stream_tty(*cmd, &block)
+      raise CommandError.new(cmd, result.status, "") unless result.success?
+
+      result
+    end
+
     # Copy a host path into a container at dest (a full destination path).
     def cp_into(name, src, dest)
       @shell.run!("docker", "cp", src, "#{name}:#{dest}")

data/lib/gdkbox/provisioner.rb

@@ -141,6 +141,37 @@ module GDKBox
       true
     BASH
 
+    # Prepares the box's SSH client for real git traffic against a remote
+    # host (issue #6, hit when a gitlab_remote is configured):
+    #
+    # - `StrictHostKeyChecking accept-new`: a fresh box has an empty
+    #   known_hosts, so the first fetch would die on host key verification.
+    # - Connection multiplexing: lazy fetches from the treeless clone open
+    #   SSH connections in bursts; gitlab.com throttles that, which surfaces
+    #   as a misleading "Permission denied (publickey)". One shared control
+    #   connection absorbs the burst.
+    #
+    # The stanza is tagged per host, so re-runs are no-ops and different
+    # remotes each get their own block.
+    GIT_SSH_SETUP = <<~'BASH'
+      set -e
+      mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
+      config="$HOME/.ssh/config"
+      touch "$config" && chmod 600 "$config"
+      if ! grep -qF "gdkbox:git-ssh $GDKBOX_GIT_HOST" "$config"; then
+        cat >> "$config" <<EOF
+
+      # gdkbox:git-ssh $GDKBOX_GIT_HOST
+      Host $GDKBOX_GIT_HOST
+        StrictHostKeyChecking accept-new
+        ControlMaster auto
+        ControlPath ~/.ssh/cm-%r@%h-%p
+        ControlPersist 20m
+        ServerAliveInterval 30
+      EOF
+      fi
+    BASH
+
     # Points the GitLab checkout's origin at a different repository. The image
     # hardcodes the community mirror (gitlab-community/gitlab-org/gitlab);
     # team members typically want the canonical gitlab-org/gitlab. The mirror
@@ -152,6 +183,27 @@ module GDKBox
       git fetch --quiet origin
     BASH
 
+    # Backfills the treeless GitLab clone (the image clones with
+    # --filter=tree:0, so operations spanning a large history gap storm the
+    # network with lazy fetches and fail; see #7). Two levels:
+    #
+    # - GDKBOX_HYDRATE_FILTER=blob:none — fetch all missing trees, keep file
+    #   contents lazy. A fraction of the download; enough for deep rebases,
+    #   which mostly storm tree fetches.
+    # - no filter (full) — fetch everything and drop the promisor config;
+    #   the repository behaves like a normal full clone afterwards.
+    HYDRATE = <<~'BASH'
+      set -e
+      if [ -n "$GDKBOX_HYDRATE_FILTER" ]; then
+        git config remote.origin.partialclonefilter "$GDKBOX_HYDRATE_FILTER"
+        git fetch --refetch --progress origin
+      else
+        git config --unset-all remote.origin.partialclonefilter || true
+        git fetch --refetch --progress origin
+        git config --unset remote.origin.promisor || true
+      fi
+    BASH
+
     def initialize(docker:, config:)
       @docker = docker
       @config = config
@@ -200,6 +252,26 @@ module GDKBox
       )
     end
 
+    # Hydrate the GitLab checkout, streaming git's progress lines to the
+    # block. trees: true fetches trees only (blobs stay lazy).
+    def hydrate(container_name, trees: false, &block)
+      @docker.exec_stream(
+        container_name, HYDRATE,
+        user: @config.ssh_user,
+        workdir: @config.gitlab_checkout_path,
+        env: { "GDKBOX_HYDRATE_FILTER" => (trees ? "blob:none" : "") },
+        &block
+      )
+    end
+
+    def setup_git_ssh(container_name, host)
+      @docker.exec(
+        container_name, GIT_SSH_SETUP,
+        user: @config.ssh_user,
+        env: { "GDKBOX_GIT_HOST" => host }
+      )
+    end
+
     def setup_gitlab_remote(container_name, url)
       @docker.exec(
         container_name, GITLAB_REMOTE_SETUP,

data/lib/gdkbox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GDKBox
-  VERSION = "0.1.10"
+  VERSION = "0.1.12"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gdkbox
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.12
 platform: ruby
 authors:
 - jotolo