@automattic/vip 4.1.3 → 4.1.5

package/docs/RELEASING.md

@@ -87,8 +87,9 @@ To publish a prerelease:
 4. The workflow creates the tag at the selected full commit and creates a draft
    release. The tag triggers Buildkite's native signing jobs.
 5. Wait while the workflow finds the Buildkite build with that exact tag and
-   commit, downloads all five archives and checksums, validates the archive
-   contents, and uploads the verified files.
+   commit, downloads all five archives, any available native installers, and their
+   checksums, validates the archive contents and installer containers, and
+   uploads the verified files. See [native installers](BUILD-SIGNING.md#native-installers).
 6. The release becomes public only after all verification and uploads succeed.
 
 If a run fails, its tag and draft release remain in place. Fix the cause and

package/docs/SETUP.md

@@ -98,7 +98,7 @@ This application uses environmental variables for vital configuration informatio
 TODO: Update description of the variables.
 
 - `API_HOST`: HTTP endpoint to use rather than the default. [For internal VIP use](TESTING.md#local-testing).
-- `DEBUG API_HOST`:
+- `DEBUG`: Enable diagnostic namespaces in the Node.js and Go CLIs, for example `DEBUG='@automattic/vip:http*'`. See [debugging](DEBUGGING.md).
 - `DOCKER_CERT_PATH`:
 - `DOCKER_CLIENT_TIMEOUT`:
 - `DOCKER_HOST`:
@@ -107,6 +107,7 @@ TODO: Update description of the variables.
 - `SOCKS_PROXY`:
 - `HTTP_PROXY`:
 - `NO_PROXY`:
+- `VIP_INSTALLER_E2E`: Set to `1` only for native installer lifecycle tests on disposable GitHub-hosted runners. The tests also require `GITHUB_ACTIONS=true` and `RUNNER_ENVIRONMENT=github-hosted`; they install and uninstall system-wide fixture packages. See [installer tests](BUILD-SIGNING.md#installer-tests).
 - `VIP_NO_UPDATE_NOTIFIER`: Set to `1` to disable automatic update checks and notices in the Go CLI. Explicit `vip-next update` commands remain available. See [Go CLI updates](GO-UPDATES.md).
 - `VIP_PROXY`: [For internal VIP use](TESTING.md#local-testing).
 - `VIP_USE_SYSTEM_PROXY`:

package/fastlane/Fastfile

@@ -9,6 +9,7 @@ PROJECT_ROOT_FOLDER = File.dirname(File.expand_path(__dir__))
 
 APPLE_TEAM_ID = 'PZYM8XX95Q'
 SIGNING_IDENTITY = "Developer ID Application: Automattic, Inc. (#{APPLE_TEAM_ID})"
+INSTALLER_SIGNING_IDENTITY = "Developer ID Installer: Automattic, Inc. (#{APPLE_TEAM_ID})"
 
 # The signature's designated requirement embeds this identifier, and the Keychain ACL
 # guarding the master key is bound to that requirement. `codesign` otherwise derives it
@@ -38,7 +39,7 @@ before_all do
   setup_ci
 end
 
-# Fetch the Developer ID Application certificate into the keychain.
+# Fetch only the Developer ID Application certificate into the keychain.
 #
 # @param readonly [Boolean] Use `true` to only fetch the existing certificate from S3 via
 #   `match`. Use `false` to create or renew it on App Store Connect.
@@ -60,6 +61,59 @@ lane :configure_code_signing do |readonly: true|
   )
 end
 
+# Fetch the Developer ID Installer certificate into the keychain.
+#
+# Separate from `configure_code_signing` so a missing Installer certificate cannot stop the
+# binary jobs publishing portable archives.
+#
+# Do not collapse this into match's top-level `type: 'developer_id_installer'`: match maps only
+# `developer_id_application` through to the certificate generator, so the installer type
+# silently resolves to an Apple Distribution certificate (`match/lib/match/generator.rb`).
+#
+# Readonly with no override, because creating this certificate needs Apple Account Holder
+# access that an App Store Connect API key does not grant. Provision it out of band with
+# `fastlane match import --type developer_id_installer`.
+#
+lane :configure_installer_signing do
+  require_env_vars!(*CODE_SIGNING_ENV_VARS)
+
+  sync_code_signing(
+    type: 'developer_id',
+    additional_cert_types: ['developer_id_installer'],
+    platform: 'macos',
+    team_id: APPLE_TEAM_ID,
+    app_identifier: [],
+    skip_provisioning_profiles: true,
+    readonly: true,
+    **CODE_SIGNING_STORAGE_OPTIONS
+  )
+
+  verify_installer_identity!
+end
+
+# The PKG has its own signing identity and stapled notarization ticket. Fetch
+# the Installer certificate via configure_installer_signing before invoking this.
+lane :sign_and_notarize_installer do |package:|
+  package = resolve_from_project_root(package)
+  UI.user_error!("No installer at '#{package}'") unless File.file?(package)
+  require_env_vars!(*ASC_API_KEY_ENV_VARS)
+  signed = "#{package}.signed.pkg"
+  begin
+    sh('productsign', '--sign', INSTALLER_SIGNING_IDENTITY, '--timestamp', package, signed)
+    details = sh('pkgutil', '--check-signature', signed, log: false)
+    unless details.include?(INSTALLER_SIGNING_IDENTITY)
+      UI.user_error!("Installer is not signed by '#{INSTALLER_SIGNING_IDENTITY}'")
+    end
+    notarize(package: signed, skip_stapling: true, api_key: app_store_connect_api_key, print_log: true)
+    sh('xcrun', 'stapler', 'staple', signed)
+    sh('xcrun', 'stapler', 'validate', signed)
+    sh('spctl', '--assess', '--type', 'install', '--verbose=2', signed)
+    FileUtils.mv(signed, package)
+  ensure
+    FileUtils.rm_f(signed)
+  end
+end
+
 # Sign a built CLI binary with the Developer ID certificate and submit it for notarization.
 #
 # Run `configure_code_signing` first, or run this on a machine that already holds the
@@ -92,8 +146,8 @@ lane :sign_and_notarize do |binary:|
 
   notarize(
     package: archive,
-    # Only bundles and disk images can carry a stapled ticket. A bare binary relies on
-    # the online Gatekeeper check.
+    # A bare binary cannot carry a stapled ticket and relies on the online
+    # Gatekeeper check. The separate PKG lane staples the installer instead.
     skip_stapling: true,
     api_key: app_store_connect_api_key,
     print_log: true
@@ -135,6 +189,15 @@ def resolve_from_project_root(path)
   File.absolute_path(path, PROJECT_ROOT_FOLDER)
 end
 
+# `match` reports success for whatever certificate storage happens to hold, so the identity
+# `productsign` needs is verified rather than assumed.
+def verify_installer_identity!
+  identities = sh('security', 'find-identity', '-v', log: false)
+  return if identities.include?(INSTALLER_SIGNING_IDENTITY)
+
+  UI.user_error!("No '#{INSTALLER_SIGNING_IDENTITY}' identity in the keychain:\n#{identities}")
+end
+
 def require_env_vars!(*names)
   missing = names.select { |name| ENV[name].to_s.empty? }
   UI.user_error!("Missing required environment variable(s): #{missing.join(', ')}") unless missing.empty?

package/fastlane/test_signing.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+
+# Evaluate the actual Fastfile DSL without accessing credentials or Apple/S3.
+module FastlaneCore
+  module Helper
+    def self.bundler?
+      true
+    end
+  end
+end
+
+module UI
+  def self.user_error!(message)
+    raise message
+  end
+end
+
+class SigningConfigurationTest < Minitest::Test
+  def setup
+    @lanes = {}
+    @calls = []
+    @dsl = Object.new
+    lanes = @lanes
+    calls = @calls
+    @dsl.define_singleton_method(:before_all) { |&block| }
+    @dsl.define_singleton_method(:lane) { |name, &block| lanes[name] = block }
+    @dsl.define_singleton_method(:sync_code_signing) do |**options|
+      calls << options
+      raise 'Installer certificate unavailable' if options[:type] == 'developer_id_installer' || options[:additional_cert_types]
+    end
+    @dsl.instance_eval(File.read(File.join(__dir__, 'Fastfile')), File.join(__dir__, 'Fastfile'))
+    @dsl.define_singleton_method(:require_env_vars!) { |*names| }
+  end
+
+  def test_missing_installer_identity_does_not_affect_application_lane
+    @lanes.fetch(:configure_code_signing).call
+    assert_equal 'developer_id', @calls.last[:type]
+    assert_equal true, @calls.last[:readonly]
+    assert_nil @calls.last[:additional_cert_types]
+    error = assert_raises(RuntimeError) { @lanes.fetch(:configure_installer_signing).call }
+    assert_equal 'Installer certificate unavailable', error.message
+    assert_equal true, @calls.last[:readonly]
+    assert_equal true, @calls.last[:skip_provisioning_profiles]
+    @lanes.fetch(:configure_code_signing).call
+  end
+
+  # Match's top-level `developer_id_installer` type silently resolves to an Apple Distribution
+  # certificate, so the supported route is pinned rather than left to preference.
+  def test_installer_lane_requests_the_installer_certificate_as_an_additional_type
+    assert_raises(RuntimeError) { @lanes.fetch(:configure_installer_signing).call }
+
+    assert_equal 'developer_id', @calls.last[:type]
+    assert_equal ['developer_id_installer'], @calls.last[:additional_cert_types]
+  end
+
+  def test_installer_lane_never_creates_certificates
+    assert_raises(RuntimeError) { @lanes.fetch(:configure_installer_signing).call }
+
+    assert_equal true, @calls.last[:readonly]
+    assert_nil @calls.last[:api_key]
+  end
+
+  def test_identity_check_accepts_the_installer_identity
+    stub_identities('  1) ABC123 "Developer ID Installer: Automattic, Inc. (PZYM8XX95Q)"')
+
+    @dsl.send(:verify_installer_identity!)
+  end
+
+  def test_identity_check_rejects_a_substituted_certificate
+    stub_identities('  1) ABC123 "Apple Distribution: Automattic, Inc. (PZYM8XX95Q)"')
+
+    error = assert_raises(RuntimeError) { @dsl.send(:verify_installer_identity!) }
+    assert_includes error.message, 'Developer ID Installer: Automattic, Inc. (PZYM8XX95Q)'
+  end
+
+  private
+
+  def stub_identities(output)
+    @dsl.define_singleton_method(:sh) { |*_args, **_options| output }
+  end
+end

package/internal/auth/logout.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/Automattic/vip/internal/debuglog"
 	"github.com/Automattic/vip/internal/httpproxy"
 )
 
@@ -13,13 +14,25 @@ import (
 // ignored; only a transport failure returns a non-nil error. The caller always
 // purges the local token regardless.
 func PostLogout(apiHost, rawToken string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	return PostLogoutContext(context.Background(), apiHost, rawToken)
+}
+
+// PostLogoutContext preserves invocation diagnostics and cancellation while
+// retaining the bounded, best-effort server-side invalidation behavior.
+func PostLogoutContext(ctx context.Context, apiHost, rawToken string) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiHost+"/logout", nil)
 	if err != nil {
 		return err
 	}
 	req.Header.Set("Authorization", "Bearer "+rawToken)
+	// Node's HTTP helper logs the request target. Only expose the fixed logout
+	// path and target host; URL userinfo, queries and headers may carry secrets.
+	debuglog.Printf(ctx, "@automattic/vip:http", "running fetch %s://%s/logout", req.URL.Scheme, req.URL.Host)
 	// NOT http.DefaultClient: this request carries the bearer token, and Node
 	// routes /logout through api/http.ts's proxy agent. See internal/httpproxy.
 	resp, err := httpproxy.Client().Do(req)

package/internal/auth/logout_test.go

@@ -1,9 +1,14 @@
 package auth
 
 import (
+	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
+
+	"github.com/Automattic/vip/internal/debuglog"
 )
 
 func TestPostLogoutSendsBearer(t *testing.T) {
@@ -21,6 +26,47 @@ func TestPostLogoutSendsBearer(t *testing.T) {
 	}
 }
 
+func TestPostLogoutContextDiagnostics(t *testing.T) {
+	for _, key := range []string{"VIP_PROXY", "vip_proxy", "SOCKS_PROXY", "socks_proxy", "NO_PROXY", "no_proxy"} {
+		t.Setenv(key, "")
+	}
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost || r.URL.Path != "/logout" || r.Header.Get("Authorization") != "Bearer fixture-token-secret" {
+			t.Error("logout request changed")
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte("server-body-secret"))
+	}))
+	defer proxy.Close()
+	t.Setenv("VIP_USE_SYSTEM_PROXY", "1")
+	t.Setenv("HTTPS_PROXY", strings.Replace(proxy.URL, "http://", "http://proxy-user:proxy-password-secret@", 1))
+	for _, namespaces := range []string{"", "@automattic/vip:http,vip:proxy-dispatcher"} {
+		t.Run(namespaces, func(t *testing.T) {
+			var out bytes.Buffer
+			ctx := debuglog.WithLogger(context.Background(), namespaces, &out)
+			if err := PostLogoutContext(ctx, "http://url-user:url-password-secret@logout.example.test", "fixture-token-secret"); err != nil {
+				t.Fatal(err)
+			}
+			if namespaces == "" {
+				if out.Len() != 0 {
+					t.Fatalf("default diagnostics = %q", out.String())
+				}
+				return
+			}
+			for _, want := range []string{"@automattic/vip:http running fetch http://logout.example.test/logout", "vip:proxy-dispatcher Enabling fetch dispatcher proxy support using config: " + proxy.URL} {
+				if !strings.Contains(out.String(), want) {
+					t.Errorf("missing %q in %q", want, out.String())
+				}
+			}
+			for _, forbidden := range []string{"secret", "proxy-user", "url-user", "Authorization"} {
+				if strings.Contains(out.String(), forbidden) {
+					t.Errorf("diagnostics exposed %q: %q", forbidden, out.String())
+				}
+			}
+		})
+	}
+}
+
 func TestPostLogoutIgnoresServerError(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusInternalServerError)

package/internal/debuglog/debuglog.go

@@ -0,0 +1,85 @@
+// Package debuglog emits opt-in diagnostics using Node debug namespace selectors.
+// Loggers belong to an invocation context, so commands and tests cannot leak
+// their debug settings or output writers into another invocation.
+package debuglog
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+	"sync"
+	"unicode"
+)
+
+type contextKey struct{}
+
+type logger struct {
+	mu               sync.Mutex
+	out              io.Writer
+	include, exclude []*regexp.Regexp
+}
+
+// WithLogger accepts comma/whitespace-separated namespaces, '*' wildcards,
+// and '-' exclusions, matching the Node debug package's selector grammar.
+func WithLogger(ctx context.Context, namespaces string, out io.Writer) context.Context {
+	l := &logger{out: out}
+	for _, pattern := range strings.FieldsFunc(namespaces, func(r rune) bool { return r == ',' || unicode.IsSpace(r) }) {
+		exclude := strings.HasPrefix(pattern, "-")
+		pattern = strings.TrimPrefix(pattern, "-")
+		if pattern == "" {
+			continue
+		}
+		expression := "^" + strings.ReplaceAll(regexp.QuoteMeta(pattern), `\*`, ".*") + "$"
+		re := regexp.MustCompile(expression)
+		if exclude {
+			l.exclude = append(l.exclude, re)
+		} else {
+			l.include = append(l.include, re)
+		}
+	}
+	return context.WithValue(ctx, contextKey{}, l)
+}
+
+// Enabled lets callers avoid constructing expensive diagnostic metadata when
+// its namespace is disabled.
+func Enabled(ctx context.Context, namespace string) bool {
+	if ctx == nil {
+		return false
+	}
+	l, _ := ctx.Value(contextKey{}).(*logger)
+	if l == nil || l.out == nil {
+		return false
+	}
+	for _, re := range l.exclude {
+		if re.MatchString(namespace) {
+			return false
+		}
+	}
+	for _, re := range l.include {
+		if re.MatchString(namespace) {
+			return true
+		}
+	}
+	return false
+}
+
+// Printf writes one diagnostic line if namespace is enabled. Callers must pass
+// only safe fields: this function escapes terminal controls, not secrets.
+func Printf(ctx context.Context, namespace, format string, args ...any) {
+	if !Enabled(ctx, namespace) {
+		return
+	}
+	l := ctx.Value(contextKey{}).(*logger)
+	line := namespace + " " + fmt.Sprintf(format, args...)
+	line = strings.Map(func(r rune) rune {
+		if unicode.IsControl(r) {
+			return ' '
+		}
+		return r
+	}, line)
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	_, _ = fmt.Fprintln(l.out, line)
+}

package/internal/debuglog/debuglog_test.go

@@ -0,0 +1,44 @@
+package debuglog
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+)
+
+func TestNamespaceFiltering(t *testing.T) {
+	for _, tc := range []struct{ selector, want string }{
+		{"", ""},
+		{"*", "alpha one\nbeta two\n"},
+		{"alpha", "alpha one\n"},
+		{"a*,beta", "alpha one\nbeta two\n"},
+		{"*,-beta", "alpha one\n"},
+		{"-beta,*", "alpha one\n"},
+		{"alpha beta", "alpha one\nbeta two\n"},
+		{"[ab]*", ""},
+	} {
+		t.Run(tc.selector, func(t *testing.T) {
+			var out bytes.Buffer
+			ctx := WithLogger(context.Background(), tc.selector, &out)
+			Printf(ctx, "alpha", "%s", "one")
+			Printf(ctx, "beta", "%s", "two")
+			if out.String() != tc.want {
+				t.Fatalf("output = %q, want %q", out.String(), tc.want)
+			}
+		})
+	}
+}
+
+func TestLoggerIsScopedToContextAndEscapesControls(t *testing.T) {
+	var out bytes.Buffer
+	ctx := WithLogger(context.Background(), "*", &out)
+	Printf(context.Background(), "alpha", "must stay silent")
+	Printf(ctx, "alpha", "name=%s", "one\n\x1b[31mtwo")
+	if strings.Count(out.String(), "\n") != 1 || strings.Contains(out.String(), "\x1b") {
+		t.Fatalf("unsafe output: %q", out.String())
+	}
+	if !strings.Contains(out.String(), "one") || !strings.Contains(out.String(), "two") {
+		t.Fatalf("missing diagnostic: %q", out.String())
+	}
+}

package/internal/devenv/compose/view.go

@@ -144,9 +144,12 @@ func wordpressTag(d *instancedata.InstanceData) string {
 // phpFPMImagePrefix is the VIP php-fpm image repo; a bare version is appended.
 const phpFPMImagePrefix = "ghcr.io/automattic/vip-container-images/php-fpm:"
 
-// defaultPHPImage is the recommended php-fpm image when none is specified —
-// the first entry of Node's DEV_ENVIRONMENT_PHP_VERSIONS (8.2, recommended).
-const defaultPHPImage = phpFPMImagePrefix + "8.2"
+// DefaultPHPVersion is the recommended PHP version for new environments.
+// Keep this aligned with Node's DEV_ENVIRONMENT_DEFAULT_PHP_VERSION.
+const DefaultPHPVersion = "8.4"
+
+// defaultPHPImage is the recommended php-fpm image when none is specified.
+const defaultPHPImage = phpFPMImagePrefix + DefaultPHPVersion
 
 // phpImage resolves the php-fpm image from instance-data's php field, mirroring
 // Node DEV_ENVIRONMENT_PHP_VERSIONS resolution: empty -> recommended default; a

package/internal/devenv/compose/view_test.go

@@ -108,7 +108,7 @@ func TestNewViewResolvesPHPImage(t *testing.T) {
 		return &instancedata.InstanceData{SiteSlug: "e", Multisite: json.RawMessage("false"), PHP: php}
 	}
 	cases := []struct{ php, want string }{
-		{"", "ghcr.io/automattic/vip-container-images/php-fpm:8.2"},                                                    // empty -> recommended default
+		{"", "ghcr.io/automattic/vip-container-images/php-fpm:8.4"},                                                    // empty -> recommended default
 		{"8.4", "ghcr.io/automattic/vip-container-images/php-fpm:8.4"},                                                 // bare version -> mapped image
 		{"ghcr.io/automattic/vip-container-images/php-fpm:8.3", "ghcr.io/automattic/vip-container-images/php-fpm:8.3"}, // explicit image -> as-is
 	}

package/internal/devenv/debug.go

@@ -0,0 +1,4 @@
+package devenv
+
+// Keep the Node namespace so existing --debug filters work across runtimes.
+const debugNamespace = "@automattic/vip:bin:dev-environment"

package/internal/devenv/debug_test.go

@@ -0,0 +1,103 @@
+package devenv
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/Automattic/vip/internal/debuglog"
+	"github.com/Automattic/vip/internal/devenv/paths"
+)
+
+func fakeDebugDocker(t *testing.T) {
+	t.Helper()
+	t.Setenv("XDG_DATA_HOME", t.TempDir())
+	t.Setenv("DOCKER_HOST", "tcp://user:docker-secret@localhost:1234")
+	bin := t.TempDir()
+	t.Setenv("PATH", bin)
+	// Only this fake is reachable; even an unexpected invocation cannot reach Docker.
+	script := `#!/bin/sh
+case "$*" in
+  "compose version") exit 0 ;;
+  "compose -p running ps --format json --all") echo '{"Service":"php","State":"running","ExitCode":0}' ;;
+  "compose -p stopped ps --format json --all") echo '{"Service":"php","State":"exited","ExitCode":0}' ;;
+  "compose -p failed ps --format json --all") echo 'private-docker-error' >&2; exit 1 ;;
+  *) exit 1 ;;
+esac
+`
+	if err := os.WriteFile(filepath.Join(bin, "docker"), []byte(script), 0o700); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestListDebugExplainsStatusWithoutChangingResults(t *testing.T) {
+	fakeDebugDocker(t)
+	for _, slug := range []string{"failed", "running", "stopped"} {
+		if err := os.MkdirAll(paths.EnvironmentPath(slug), 0o700); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, ns := range []string{"@automattic/vip:bin:dev-environment", "unrelated", ""} {
+		t.Run(ns, func(t *testing.T) {
+			var out bytes.Buffer
+			got, err := List(debuglog.WithLogger(context.Background(), ns, &out))
+			want := []EnvStatus{{Slug: "failed"}, {Slug: "running", Running: true}, {Slug: "stopped"}}
+			if err != nil || !reflect.DeepEqual(got, want) {
+				t.Fatalf("List = %#v, %v", got, err)
+			}
+			if ns != "@automattic/vip:bin:dev-environment" {
+				if out.Len() != 0 {
+					t.Fatalf("unexpected diagnostics: %s", &out)
+				}
+				return
+			}
+			for _, text := range []string{"Names found", "failed", "status query failed", "running=true", "running=false"} {
+				if !strings.Contains(out.String(), text) {
+					t.Errorf("missing %q in %s", text, &out)
+				}
+			}
+			for _, secret := range []string{"docker-secret", "private-docker-error"} {
+				if strings.Contains(out.String(), secret) {
+					t.Errorf("leaked %q", secret)
+				}
+			}
+		})
+	}
+}
+
+func TestDevEnvDebugSummariesOmitArgumentsAndConfig(t *testing.T) {
+	fakeDebugDocker(t)
+	for _, tc := range []struct {
+		name string
+		run  func(context.Context) error
+	}{
+		{"create", func(ctx context.Context) error {
+			return Create(ctx, CreateConfig{Title: "secret-title", Domain: "secret-domain"})
+		}},
+		{"update", func(ctx context.Context) error {
+			value := "secret-config"
+			return Update(ctx, "missing", UpdateConfig{XdebugConfig: &value})
+		}},
+		{"exec", func(ctx context.Context) error {
+			return Exec(ctx, "missing", []string{"config", "set", "DB_PASSWORD", "secret-argument"})
+		}},
+		{"shell", func(ctx context.Context) error {
+			return Shell(ctx, "missing", "php", false, []string{"echo", "secret-argument"})
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var out bytes.Buffer
+			_ = tc.run(debuglog.WithLogger(context.Background(), "@automattic/vip:bin:dev-environment", &out))
+			if !strings.Contains(out.String(), tc.name) {
+				t.Fatalf("missing operation diagnostic: %s", &out)
+			}
+			if strings.Contains(out.String(), "secret-") || strings.Contains(out.String(), "docker-secret") {
+				t.Fatalf("secret in diagnostic: %s", &out)
+			}
+		})
+	}
+}