wslc-wip 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c125a123c76462cbb6b78fe0ead5ea400b8fd0f7a8c825421985547190603d02
-  data.tar.gz: 0762dbbf6480a84d9ef58d9481d1398672b5d2ef7e8d644d39be655558704022
+  metadata.gz: b0fdc63759c329dc9cbc9a60a0ef184db31087d454070871bc3f39a1982d7426
+  data.tar.gz: 447d4f3206b3662def93d376b950de99564f3efbfc85e8607bbaad6fb432545d
 SHA512:
-  metadata.gz: b91868415f3069167ddd479616d9bd2937d3684af2e7c6228789b198cd7af952e9d2705fe363877ef05a6831a7f5a8e9a203e073623bb0c4b46102f2771429c5
-  data.tar.gz: 88b8d3fdec972776b0bdefedd5753ebcb4a025cb9bf54122ad7eb723e663f4cac64dff5a4ec201bd45b5959b9f44ba6801e8441419ff72f388631eb7eee80f67
+  metadata.gz: fc07a96116a3c737332a313447663a1730d6d15729c329b6e121a356de2bf62ba696b7386a854c685a7bb2415fc5b7c0edc42437ec86b80b2d13de99b6075cad
+  data.tar.gz: 61b8a28cd2953335ca38a97cf15f52302aa070698d183d95fefc1edcd7d85d661cb8461ebe1de830969f4b9480734dfeb13c6431821925a500377eb56b1a4221

data/README.md

@@ -97,6 +97,11 @@ commands:
     type: build
     context: .
     tag: slidict/slidict:development
+sync: # optional; mirror the source into a named volume instead of bind-mounting it live
+  exclude:
+    - .git
+    - tmp/
+    - node_modules/
 ```
 
 `env` values are stringified. `wip config` masks any key matching token, password, secret,
@@ -129,6 +134,56 @@ the same way Compose's service names resolve. `wip down` tears the main containe
 dependencies down (the network itself is left in place). Each dependency entry accepts `image`
 (required), `command`, `env`, `ports`, `volumes`, and `workdir` — the same shape as `defaults`.
 
+### Source sync
+
+Bind-mounting the app directory (`.:/app`) is what usually makes a container boot crawl under
+wslc — see [Slow boot when the app directory is
+bind-mounted](#slow-boot-when-the-app-directory-is-bind-mounted) for why. A `sync:` block hands
+that problem to `wip`: the source is mounted read-only, the app runs off a named volume, and wip
+mirrors one into the other with `rsync`.
+
+```yaml
+sync:
+  source: .          # host path, relative to wip.yml (default: the wip.yml directory)
+  target: /app       # container path served by the volume (default: defaults.workdir, else /app)
+  volume: app-src    # named volume holding the mirror (default: "<defaults.container>-src")
+  mount: /host-src   # where the source is bind-mounted read-only (default: /host-src)
+  exclude:           # rsync --exclude patterns
+    - .git
+    - tmp/
+    - node_modules/
+  delete: true       # rsync --delete (default: true)
+  command: rsync     # binary that does the mirroring (default: rsync)
+  options: []        # extra flags appended to the rsync invocation
+  interval: 2        # seconds between syncs for `wip sync --watch` (default: 2)
+```
+
+Everything below `sync:` is optional — `sync: {}` alone already works. With it in place:
+
+- Any `defaults.volumes` entry mounting `target` (the usual `.:/app`) is replaced by
+  `<source>:/host-src:ro` plus `app-src:/app`, so the running app only ever touches the volume.
+  Other volumes (`bundle:/usr/local/bundle`, ...) are passed through untouched, and
+  `dependencies:` keep mounting whatever they declare.
+- `wip up` mirrors the source into the volume before the container boots; `wip up --no-sync`
+  skips that step.
+- `wip sync` mirrors on demand — `wslc exec`ing rsync inside the container when it's running, and
+  falling back to a throwaway container with the same mounts when it isn't.
+- `wip sync --watch [--interval N]` keeps re-syncing until Ctrl-C, so host edits reach the
+  container with a short delay. Run it in a second terminal alongside `wip up -d`.
+- `wip doctor` reports the resolved source, volume, and target, and fails if the source is missing.
+
+Like every built-in command, `wip sync` takes precedence over a `commands:` entry of the same
+name; wip says so and points at `wip dispatch sync`, which still runs yours.
+
+Two things to keep in mind. The mirror runs `rsync` *inside* the container, so the image needs it
+(`RUN apt-get update && apt-get install -y rsync`) — or point `sync.command` at a copy tool the
+image already has. And the mirror is one-way (host → volume): anything the app writes under
+`target` is removed by the next `--delete` pass unless you `exclude` it, give it its own volume,
+or set `delete: false`.
+
+`sync:` is mutually exclusive with `compose:` — in compose mode the compose file owns the volume
+layout.
+
 ### Compose mode
 
 If your project already has a real `compose.yml`, don't duplicate it in `dependencies:` — point
@@ -178,12 +233,13 @@ found, its version, and which compose file `wip` resolved.
 | `wip doctor` | Diagnose WSL2, interop, WSLC, config, architecture, and Git |
 | `wip config` | Print the effective configuration (secrets masked) |
 | `wip build -- --no-cache` | Build the image from the `build` definition |
-| `wip up [-d]` | Start `defaults.container` and its `dependencies` (creating any that are missing, on `defaults.network` if set). `-d` runs the main container in the background |
+| `wip up [-d] [--no-sync]` | Start `defaults.container` and its `dependencies` (creating any that are missing, on `defaults.network` if set). `-d` runs the main container in the background; with `sync:` configured, the source is mirrored into the volume first unless `--no-sync` |
 | `wip down` | Stop and remove `defaults.container` and its `dependencies` |
 | `wip exec [--no-interactive] COMMAND...` | Run a command in the existing container |
 | `wip run [--no-interactive] COMMAND...` | Run a command in a new `--rm` container (compose mode: `exec`s into `compose.service` instead) |
 | `wip shell` | Open the configured shell, falling back to `bash` then `sh` |
 | `wip logs [-f] [SERVICE...]` | Follow compose service logs (compose mode only) |
+| `wip sync [-w] [--interval N]` | Mirror the source into the sync volume once, or keep re-syncing with `--watch` (needs `sync:`) |
 | `wip NAME ARGS...` | Run `commands.NAME`, appending any extra arguments |
 
 TTY allocation is decided by combining the command's config, the CLI option, and whether both
@@ -261,12 +317,11 @@ side can look busy while almost no data is actually transferred, and the process
 for minutes with barely any resource usage to show for it.
 
 If a debug log shows a boot-time command "stuck" with low CPU/mem/IO in `resource_monitor`'s
-output, this is worth checking before assuming the app itself is broken. The workaround is to stop
-bind-mounting the source live and instead mirror it into a named volume: mount the host path
-read-only (`.:/host-src:ro`), add a named volume for `/app`, and have the container's entrypoint
-`rsync` from the read-only mount into the volume before boot, then keep syncing in the background
-(e.g. every couple of seconds) so edits still show up with a short delay. The app then only ever
-touches fast native storage once it's running.
+output, this is worth checking before assuming the app itself is broken. The fix is to stop
+bind-mounting the source live and mirror it into a named volume instead, so the app only ever
+touches fast native storage once it's running. `wip` does that for you — add a `sync:` block and
+it rewrites the mounts, mirrors before boot, and re-syncs on demand. See [Source
+sync](#source-sync).
 
 ### CPU architecture mismatch
 
@@ -330,10 +385,12 @@ for `wip`, roughly in priority order:
 3. **Bind-mount boot time (`rails c`, `bundle`, ...)** — commands like `wip rails c` still start
    noticeably slower than the equivalent under `docker compose`, mostly from WSL2 bind-mounted
    (`.:/app`-style) volumes doing many small reads for gems/`node_modules` (use `--debug` to
-   confirm it's disk I/O and not `wip`'s own overhead). This isn't really `wip`'s responsibility —
-   the fix has to come from either the mount layer or the project's own volume layout. We're also
-   hoping for improvements on the `wslc` side itself (faster bind-mount/cache behavior); `wip`
-   will pick those up for free as soon as they land.
+   confirm it's disk I/O and not `wip`'s own overhead). [Source sync](#source-sync) works around
+   this today by running the app off a named volume and mirroring the host tree into it, at the
+   cost of a one-way sync with a short delay. A tighter loop (host-side file watching instead of
+   interval polling, two-way sync) is the natural next step. We're also hoping for improvements on
+   the `wslc` side itself (faster bind-mount/cache behavior); `wip` will pick those up for free as
+   soon as they land.
 
 Each of these should stay additive to the existing `wip.yml` shape — no breaking changes to
 `defaults`, `commands`, `dependencies`, or `compose` are planned. A resident daemon and a GUI

data/lib/wip/cli.rb

@@ -63,6 +63,7 @@ module Wip
 
     desc 'up', 'Start the configured container and its dependencies, creating them if necessary'
     option :detach, type: :boolean, default: false, aliases: '-d'
+    option :sync, type: :boolean, default: true, desc: 'Mirror the source into the sync volume first (--no-sync skips)'
     def up
       if load_config.compose?
         return execute(compose_bridge.up(detach: options[:detach]), interactive: tty?(!options[:detach]))
@@ -70,9 +71,29 @@ module Wip
 
       ensure_network
       load_config.dependencies.each_key { |name| ensure_dependency(name) }
+      sync_before_boot if options[:sync]
       ensure_container
     end
 
+    desc 'sync', 'Mirror the source tree into the sync volume'
+    option :watch, type: :boolean, default: false, aliases: '-w', desc: 'Keep re-syncing until interrupted'
+    option :interval, type: :numeric, desc: 'Seconds between syncs when watching (default: sync.interval)'
+    def sync
+      settings = sync_settings!
+      warn_shadowed_command('sync')
+      return run_sync unless options[:watch]
+
+      interval = watch_interval(settings)
+      warn "wip: syncing #{settings.source} -> #{settings.volume}:#{settings.target} " \
+           "every #{interval}s (Ctrl-C to stop)"
+      loop do
+        run_sync(exit_on_failure: false)
+        sleep interval
+      end
+    rescue Interrupt
+      warn "\nwip: sync stopped"
+    end
+
     desc 'down', 'Stop and remove the configured container and its dependencies'
     def down
       return execute(compose_bridge.down, exit_on_failure: false) if load_config.compose?
@@ -228,6 +249,46 @@ module Wip
       end
     end
 
+    def sync_settings!
+      load_config.sync || raise(ConfigError, '`wip sync` needs a sync: block in wip.yml')
+    end
+
+    # `sync.interval` is validated when the config loads; --interval isn't, and
+    # a negative one would only surface as an ArgumentError from `sleep`.
+    def watch_interval(settings)
+      interval = options[:interval] || settings.interval
+      raise ConfigError, '--interval must be a positive number' unless interval.positive?
+
+      interval
+    end
+
+    # A built-in command wins over a `commands:` entry of the same name, so
+    # point at `wip dispatch` rather than letting the custom one vanish.
+    def warn_shadowed_command(name)
+      return unless load_config.commands.key?(name)
+
+      warn "wip: commands.#{name} in wip.yml is shadowed by the built-in `wip #{name}`; " \
+           "run it with `wip dispatch #{name}`"
+    end
+
+    # Inside the running container the mirror is a plain `exec`; otherwise it
+    # takes a throwaway container that mounts the same source and volume.
+    def run_sync(exit_on_failure: true)
+      command = container_running? ? builder.sync_exec : builder.sync_run
+      execute(command, exit_on_failure: exit_on_failure)
+    end
+
+    def container_running? = resource_exists?(builder.find_running)
+
+    def sync_before_boot
+      settings = load_config.sync
+      return unless settings
+
+      warn "wip: syncing #{settings.source} -> #{settings.volume}:#{settings.target}"
+      execute(builder.sync_run)
+      warn "wip: run `wip sync --watch` in another terminal to keep #{settings.target} up to date"
+    end
+
     def ensure_container
       container = load_config.defaults['container']
       interactive = tty?(!options[:detach])

data/lib/wip/command_builder.rb

@@ -49,6 +49,26 @@ module Wip
       [@wslc, 'list', '--all', '--filter', "name=#{container}", '--format', 'json']
     end
 
+    def find_running
+      container = required(@config.defaults, 'container')
+      [@wslc, 'list', '--filter', "name=#{container}", '--format', 'json']
+    end
+
+    # Mirrors into the volume from a throwaway container, for when the app
+    # container isn't running yet (or at all) — e.g. just before `up` boots it.
+    def sync_run
+      sync = required_sync
+      command = [@wslc, 'run', '--rm']
+      sync.volume_specs.each { |spec| command.push('-v', spec) }
+      command.push(required(@config.defaults, 'image')).concat(sync.mirror_command)
+    end
+
+    # Mirrors from inside the running container, which already has both the
+    # read-only source mount and the volume attached.
+    def sync_exec
+      [@wslc, 'exec', required(@config.defaults, 'container'), *required_sync.mirror_command]
+    end
+
     def down
       [@wslc, 'stop', required(@config.defaults, 'container')]
     end
@@ -70,7 +90,7 @@ module Wip
       command = [@wslc, 'run', '--name', name.to_s]
       command.push('--network', @config.network) if @config.network
       command << '-d' if detach
-      command.concat(options(values)).push(required(values, 'image'))
+      command.concat(options(values, sync: false)).push(required(values, 'image'))
       command.concat(Shellwords.split(values['command'].to_s)) unless values['command'].to_s.empty?
       command
     end
@@ -113,18 +133,29 @@ module Wip
 
     private
 
-    def options(values, include_container: false, include_publish: true)
+    def options(values, include_container: false, include_publish: true, sync: true)
       result = []
       result.push('-w', values['workdir']) unless values['workdir'].to_s.empty?
       merged_env(values).each { |key, value| result.push('-e', "#{key}=#{value}") }
       if include_publish
         Array(values['ports']).each { |port| result.push('-p', port.to_s) }
-        Array(values['volumes']).each { |volume| result.push('-v', volume.to_s) }
+        volume_specs(values, sync: sync).each { |volume| result.push('-v', volume) }
       end
       result << required(values, 'container') if include_container
       result
     end
 
+    # With sync configured, a live bind mount of the target (`.:/app`) is
+    # swapped for the read-only source mount plus the named volume, so the
+    # running app only ever touches the volume.
+    def volume_specs(values, sync: true)
+      specs = Array(values['volumes']).map(&:to_s)
+      settings = sync ? @config.sync : nil
+      return specs unless settings
+
+      specs.reject { |spec| settings.replaces?(spec) } + settings.volume_specs
+    end
+
     # .env supplies defaults; env set in wip.yml (defaults or per-command) wins on conflict.
     def merged_env(values) = @dotenv.merge(values.fetch('env', {}))
 
@@ -142,6 +173,10 @@ module Wip
       network
     end
 
+    def required_sync
+      @config.sync || raise(ConfigError, 'No sync: block configured in wip.yml')
+    end
+
     def dependency_values(name)
       @config.dependency(name) || raise(ConfigError, "Unknown dependency: #{name}")
     end

data/lib/wip/config.rb

@@ -26,6 +26,13 @@ module Wip
     def compose_file = compose && compose['file']
     def compose_project = compose && compose['project']
     def compose_command = compose && compose['command']
+    def sync? = !!sync
+
+    def sync
+      return @sync if defined?(@sync)
+
+      @sync = @raw.key?('sync') ? build_sync : nil
+    end
 
     def command(name)
       entry = commands[name.to_s]
@@ -44,6 +51,7 @@ module Wip
     def to_h(redact: true)
       value = { 'version' => 1, 'wslc' => { 'command' => wslc_command }, 'defaults' => defaults,
                 'up' => { 'command' => up_command }, 'dependencies' => dependencies, 'compose' => compose,
+                'sync' => sync&.to_h,
                 'commands' => commands.transform_values { |entry| defaults.merge('type' => 'exec').merge(entry) } }
       redact ? redact_secrets(value) : value
     end
@@ -57,6 +65,17 @@ module Wip
       validate_commands!
       validate_dependencies!
       validate_compose!
+      validate_sync!
+    end
+
+    def build_sync
+      SyncSettings.new(@raw['sync'], base: path && File.dirname(path.to_s),
+                                     workdir: defaults['workdir'], container: defaults['container'])
+    end
+
+    def validate_sync!
+      return unless sync
+      raise ConfigError, 'sync is mutually exclusive with compose' if compose?
     end
 
     def validate_commands!

data/lib/wip/doctor.rb

@@ -21,9 +21,7 @@ module Wip
       results << result(@environment.wsl2? ? :ok : :fail, *wsl2_messages)
       results << interop_result unless @environment.windows?
       results << Result.new(:ok, "Architecture: #{@environment.architecture}")
-      config = load_config(results)
-      check_wslc(config, results) if config
-      check_compose(config, results) if config&.compose?
+      check_config(load_config(results), results)
       results << result(command_available?('git') ? :ok : :warn, 'Git is available',
                         'Git is not available to the WSLC build environment')
       results
@@ -53,6 +51,14 @@ module Wip
       nil
     end
 
+    def check_config(config, results)
+      return unless config
+
+      check_wslc(config, results)
+      check_compose(config, results) if config.compose?
+      check_sync(config, results) if config.sync?
+    end
+
     def check_wslc(config, results)
       command = resolve(config, results)
       check_version(command, results) if command
@@ -98,6 +104,13 @@ module Wip
       results << Result.new(:fail, e.message)
     end
 
+    def check_sync(config, results)
+      sync = config.sync
+      results << result(File.directory?(sync.source) ? :ok : :fail,
+                        "Sync source #{sync.source} mirrors into volume #{sync.volume} at #{sync.target}",
+                        "Sync source not found: #{sync.source}")
+    end
+
     def result(condition, ok_message, fail_message)
       Result.new(condition,
                  condition == :ok ? ok_message : fail_message)

data/lib/wip/error_interpreter.rb

@@ -3,6 +3,14 @@
 module Wip
   # Translates raw WSLC error output into friendlier hints.
   class ErrorInterpreter
+    # Shells report a missing rsync as "rsync: not found", while the container
+    # runtime names the executable either before or after its own phrasing.
+    RSYNC_MISSING = Regexp.union(
+      /rsync: (?:command )?not found/i,
+      /rsync[^\n]*executable file not found/i,
+      /executable file not found[^\n]*rsync/i
+    )
+
     def initialize(architecture: Environment.new.architecture)
       @architecture = architecture
     end
@@ -11,11 +19,24 @@ module Wip
       case output
       when /pull access denied|insufficient_scope|authorization failed/i then registry_message
       when %r{no matching manifest for linux/(?:amd64|arm64)}i then architecture_message
+      when RSYNC_MISSING then rsync_message
       end
     end
 
     private
 
+    def rsync_message
+      <<~TEXT
+        `wip sync` needs rsync inside the image.
+
+        Install it in your Dockerfile:
+
+          RUN apt-get update && apt-get install -y rsync
+
+        Or point sync.command at a tool the image already has.
+      TEXT
+    end
+
     def registry_message
       <<~TEXT
         The container registry rejected the request.

data/lib/wip/sync_settings.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Wip
+  # Validated access to the `sync:` block, which mirrors the host source tree
+  # into a named volume instead of bind-mounting it live.
+  #
+  # A bind-mounted app directory is shared into the container's VM over
+  # virtiofs, so every stat/open a boot-time directory scan makes is a round
+  # trip. Mirroring the tree into a named volume (native storage inside the VM)
+  # and re-running the mirror on demand keeps the edit-on-the-host workflow
+  # while leaving the running app on fast disk.
+  class SyncSettings
+    DEFAULT_MOUNT = '/host-src'
+    DEFAULT_TARGET = '/app'
+    DEFAULT_BINARY = 'rsync'
+    DEFAULT_INTERVAL = 2
+    # -a preserves modes and timestamps so file watchers and bundler see a
+    # faithful copy rather than a tree that looks freshly written every sync.
+    BASE_OPTIONS = %w[-a].freeze
+    # Trailing mount options wslc/docker accept after the container path.
+    VOLUME_MODES = %w[ro rw z Z cached delegated consistent].freeze
+
+    attr_reader :target, :mount, :volume, :exclude, :binary, :extra_options, :interval
+
+    def initialize(raw, base: nil, workdir: nil, container: nil)
+      raise ConfigError, 'sync must be a mapping' unless raw.is_a?(Hash)
+
+      @base = base
+      assign_paths(raw, workdir: workdir, container: container)
+      assign_mirror(raw)
+      validate!
+    end
+
+    def delete? = !!@delete
+
+    # Expanded against the wip.yml directory so the mirror covers the same tree
+    # no matter which subdirectory wip was invoked from.
+    def source
+      @source ||= @base ? Pathname(@base).join(@raw_source).expand_path.to_s : @raw_source
+    end
+
+    # What `-v` specs the main container needs: the source read-only, and the
+    # named volume where the app actually runs.
+    def volume_specs = ["#{source}:#{mount}:ro", "#{volume}:#{target}"]
+
+    # True for a configured volume that sync replaces, so `.:/app` in
+    # `defaults.volumes` quietly becomes the read-only mount plus the volume.
+    def replaces?(spec)
+      [target.chomp('/'), mount.chomp('/')].include?(container_path(spec))
+    end
+
+    # Trailing slashes matter to rsync: they copy the *contents* of the mount
+    # into the target rather than nesting it one directory deeper.
+    def mirror_command
+      command = [binary, *BASE_OPTIONS]
+      command << '--delete' if delete?
+      command.concat(exclude.map { |pattern| "--exclude=#{pattern}" })
+      command.concat(extra_options)
+      command.push("#{mount.chomp('/')}/", "#{target.chomp('/')}/")
+    end
+
+    def to_h
+      { 'source' => source, 'target' => target, 'mount' => mount, 'volume' => volume,
+        'delete' => delete?, 'exclude' => exclude, 'command' => binary, 'options' => extra_options,
+        'interval' => interval }
+    end
+
+    private
+
+    def assign_paths(raw, workdir:, container:)
+      @raw_source = presence(raw['source']) || '.'
+      @target = presence(raw['target']) || presence(workdir) || DEFAULT_TARGET
+      @mount = presence(raw['mount']) || DEFAULT_MOUNT
+      @volume = presence(raw['volume']) || "#{presence(container) || 'wip'}-src"
+    end
+
+    def assign_mirror(raw)
+      @delete = raw.fetch('delete', true)
+      @exclude = Array(raw['exclude']).map(&:to_s)
+      @binary = presence(raw['command']) || DEFAULT_BINARY
+      @extra_options = Array(raw['options']).map(&:to_s)
+      @interval = raw.key?('interval') ? raw['interval'] : DEFAULT_INTERVAL
+    end
+
+    def validate!
+      raise ConfigError, 'sync.target must be an absolute path' unless @target.start_with?('/')
+      raise ConfigError, 'sync.mount must be an absolute path' unless @mount.start_with?('/')
+      raise ConfigError, 'sync.mount must differ from sync.target' if @mount.chomp('/') == @target.chomp('/')
+      return if @interval.is_a?(Numeric) && @interval.positive?
+
+      raise ConfigError, 'sync.interval must be a positive number'
+    end
+
+    def container_path(spec)
+      parts = spec.to_s.split(':')
+      parts.pop if parts.size > 2 && VOLUME_MODES.include?(parts.last)
+      parts.last.to_s.chomp('/')
+    end
+
+    def presence(value) = value.to_s.empty? ? nil : value.to_s
+  end
+end

data/lib/wip/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Wip
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

data/lib/wip.rb

@@ -2,6 +2,7 @@
 
 require_relative 'wip/version'
 require_relative 'wip/errors'
+require_relative 'wip/sync_settings'
 require_relative 'wip/config'
 require_relative 'wip/config_loader'
 require_relative 'wip/environment'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wslc-wip
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Wip contributors
@@ -50,6 +50,7 @@ files:
 - lib/wip/error_interpreter.rb
 - lib/wip/errors.rb
 - lib/wip/resource_monitor.rb
+- lib/wip/sync_settings.rb
 - lib/wip/version.rb
 homepage: https://wslc-wip.slidict.com/
 licenses: