conlink 2.2.0 → 2.5.0

package/net2dot.cljs

@@ -29,7 +29,7 @@
       (S/replace #"[*]" "_STAR_")
       (S/replace #"[$]" "_DOLLAR_")
       (S/replace #"[{]" "_LCURLY_")
-      (S/replace #"[}]" "_LCURLY_")
+      (S/replace #"[}]" "_RCURLY_")
       (S/replace #"[ ]" "_SPACE_")))
 
 (defn node-props [label props]
@@ -61,7 +61,7 @@
                   #(->> (subgraph conlink (str "cluster_bridge_" %2)
                                   %2 BRIDGE-PROPS)
                         (assoc %1 %2))
-                  {} (keep :bridge (:links network-config)))
+                  {} (map :bridge (keep :bridge (:links network-config))))
         services (reduce
                    #(->> (subgraph host (str "cluster_service_" (dot-id %2))
                                    (str "service '" (name %2) "'") SVC-PROPS)
@@ -85,7 +85,7 @@
                                    "-" (name dev)))
                 out-id (str "out-" outer-dev)
                 out-parent (condp = (keyword base)
-                             :conlink (get bridges bridge)
+                             :conlink (get bridges (:bridge bridge))
                              :host host)
                 {:keys [type vlanid]} link
                 [elabel iprops] (if (= "host" base)

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "conlink",
-  "version": "2.2.0",
+  "version": "2.5.0",
   "description": "conlink -  Declarative Low-Level Networking for Containers",
   "repository": "https://github.com/LonoCloud/conlink",
   "license": "SEE LICENSE IN LICENSE",
   "dependencies": {
-    "@lonocloud/resolve-deps": "^0.0.2",
+    "@lonocloud/resolve-deps": "^0.1.0",
     "ajv": "^8.12.0",
     "dockerode": "^3.3.4",
     "nbb": "^1.2.179",
@@ -14,6 +14,7 @@
     "yaml": "^2.2.1"
   },
   "devDependencies": {
+    "@lonocloud/dctest": "0.1.1",
     "shadow-cljs": "^2.25.7",
     "source-map-support": "^0.5.21"
   }

package/schema.yaml

@@ -3,6 +3,7 @@ $defs:
   ip:   {type: string, pattern: "^([0-9]{1,3}[.]){3}[0-9]+$"}
   mac:  {type: string, pattern: "^([0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$"}
   intf: {type: string, pattern: "^.{1,15}$"}
+  fwd:  {type: string, pattern: "^[0-9]{1,5}:[0-9]{1,5}/(tcp|udp)$"}
 
 type: object
 additionalProperties: false
@@ -40,14 +41,21 @@ properties:
         ip:        { "$ref": "#/$defs/cidr" }
         mac:       { "$ref": "#/$defs/mac" }
         mtu:       {type: number}
-        route:     {type: string}
         nat:       { "$ref": "#/$defs/ip" }
-        netem:     {type: string}
         mode:      {type: string}
         vlanid:    {type: number}
+        route:
+          oneOf: [{type: string},
+                  {type: array, items: {type: string}}]
         forward:
-          type: array
-          items: {type: string, pattern: "^[0-9]{1,5}:[0-9]{1,5}/(tcp|udp)$"}
+          oneOf: [{ "$ref": "#/$defs/fwd" },
+                  {type: array, items: { "$ref": "#/$defs/fwd" }}]
+        netem:
+          oneOf: [{type: string},
+                  {type: array, items: {type: string}}]
+        ethtool:
+          oneOf: [{type: string},
+                  {type: array, items: {type: string}}]
 
   bridges:
     type: array
@@ -66,7 +74,9 @@ properties:
         bridge:    {type: string}
         remote:    { "$ref": "#/$defs/ip" }
         vni:       {type: number}
-        netem:     {type: string}
+        netem:
+          oneOf: [{type: string},
+                  {type: array, items: {type: string}}]
 
   commands:
     type: array

package/scripts/copy.sh

@@ -46,7 +46,8 @@ dst_dir="${1}"; shift || die 2 "Usage: ${0} [-T|--template] SRC_DIR DST_DIR"
 done
 
 if [ "${*}" ]; then
-    exec "${@}"
+  echo "Running: ${*}"
+  exec "${@}"
 else
-    true
+  true
 fi

package/scripts/wait.sh

@@ -69,5 +69,5 @@ done
 
 if [ "${*}" ]; then
   echo "Running: ${*}"
-  exec ${*}
+  exec "${@}"
 fi

package/src/conlink/core.cljs

@@ -32,6 +32,8 @@ General Options:
                                     [default: auto] [env: CONLINK_BRIDGE_MODE]
   --default-mtu MTU                 Default link MTU (for non *vlan types)
                                     [default: 65535]
+  --keep-veth-offload               Do not add '--offload tx off' as the first
+                                    ethtool setting for container veth interfaces
   --network-file NETWORK-FILE...    Network config file
   --compose-file COMPOSE-FILE...    Docker compose file with network config
   --compose-project NAME            Docker compose project name for resolving
@@ -57,7 +59,8 @@ General Options:
                         " --system-id=random --no-mlockall --delete-bridges"))
 
 (def VLAN-TYPES #{:vlan :macvlan :macvtap :ipvlan :ipvtap})
-(def LINK-ADD-OPTS [:ip :mac :route :mtu :nat :netem :mode :vlanid :remote :vni])
+(def LINK-ADD-OPTS [:ip :mac :route :mtu :nat :netem :ethtool
+                    :mode :vlanid :remote :vni])
 (def INTF-MAX-LEN 15)
 (def DOCKER-INTF "DOCKER-ETH0")
 
@@ -77,6 +80,11 @@ General Options:
 (defn indent-pprint-str [o pre]
   (indent (trim (with-out-str (pprint o))) pre))
 
+(defn random-mac
+  "Return a random MAC address starting with '0xc2'"
+  []
+  (addrs/int->mac (reduce #(+ (rand-int 256) (* %1 256))
+                          0xc2 [1 2 3 4 5])))
 
 (defn load-configs
   "Load network configs from a list of compose file paths and a list
@@ -104,31 +112,49 @@ General Options:
   - Add default values to a link:
     - type: veth
     - dev: eth0
+    - mac: random MAC starting with first octet of 'c2'
     - mtu: --default-mtu (for non *vlan type)
     - base: :conlink for veth type, :host for *vlan types, :local otherwise"
-  [{:as link :keys [type base bridge ip forward]} bridges]
-  (let [{:keys [default-mtu docker-eth0?]} @ctx
+  [{:as link :keys [type bridge ip route forward netem ethtool]} bridges opts]
+  (let [{:keys [docker-eth0? docker-eth0-address]} @ctx
+        {:keys [default-mtu keep-veth-offload]} opts
         type (keyword (or type "veth"))
         dev (get link :dev "eth0")
+        mac (get link :mac (random-mac))
         base-default (cond (= :veth type)     :conlink
                            (VLAN-TYPES type)  :host
                            :else              :local)
         base (get link :base base-default)
         bridge (get bridges bridge)
+        route (if (string? route) [route] route)
+        forward (if (string? forward) [forward] forward)
+        netem (if (string? netem) [netem] netem)
+        ethtool-pre (if (and (= :veth type) (not keep-veth-offload))
+                      ["--offload tx off"]
+                      [])
+        ethtool (into ethtool-pre (if (string? ethtool) [ethtool] ethtool))
         link (merge
                link
                {:type type
                 :dev  dev
-                :base base}
+                :base base
+                :mac mac
+                :ethtool ethtool}
                (when bridge
                  {:bridge bridge})
                (when (not (VLAN-TYPES type))
                  {:mtu  (get link :mtu default-mtu)})
+               (when route
+                 {:route route})
                (when forward
                  {:forward
                   (map #(let [[port_a port_b proto] (S/split % #"[:/]")]
                           [(js/parseInt port_a) (js/parseInt port_b) proto])
-                       forward)}))]
+                       forward)})
+               (when (and forward docker-eth0-address)
+                 {:route (conj route (str docker-eth0-address "/32"))})
+               (when netem
+                 {:netem netem}))]
     (when forward
       (let [link-id (str (or (:service link) (:container link)) ":" dev)
             pre (str "link '" link-id "' has forward setting")]
@@ -148,8 +174,9 @@ General Options:
   is loaded otherwise fall back to :linux. Exit with an error if mode is :ovs
   or :patch and the 'openvswitch' or 'act_mirred' kernel modules are not
   loaded respectively."
-  [{:as bridge-opts :keys [bridge mode]}]
-  (let [{:keys [warn default-bridge-mode kmod-ovs? kmod-mirred?]} @ctx
+  [{:as bridge-opts :keys [bridge mode]} opts]
+  (let [{:keys [warn kmod-ovs? kmod-mirred?]} @ctx
+        {:keys [default-bridge-mode]} opts
         mode (keyword (or mode default-bridge-mode))
         _ (when (and (= :ovs mode) (not kmod-ovs?))
             (fatal 1 (str "bridge " bridge " mode is 'ovs', "
@@ -157,11 +184,11 @@ General Options:
         _ (when (and (= :patch mode) (not kmod-mirred?))
             (warn (str "bridge " bridge " mode is 'patch', "
                        "but no 'act_mirred' kernel module loaded, "
-                       " assuming it will load when needed.")))
+                       "assuming it will load when needed.")))
         _ (when (and (= :auto mode) (not kmod-ovs?))
             (warn (str "bridge " bridge " mode is 'auto', "
-                       " but no 'openvswitch' kernel module loaded, "
-                       " so falling back to 'linux'")))
+                       "but no 'openvswitch' kernel module loaded, "
+                       "so falling back to 'linux'")))
         mode (if (= :auto mode)
               (if kmod-ovs? :ovs :linux)
               mode)]
@@ -172,8 +199,9 @@ General Options:
   add :bridges, :containers, and :services maps with restructured bridge, link,
   and command configuration to provide a more efficient structure for looking
   up configuration later."
-  [{:as cfg :keys [links commands bridges]}]
-  (let [bridge-map (reduce (fn [acc b] (assoc acc (:bridge b) b))
+  [{:as cfg :keys [links bridges tunnels commands]} opts]
+  (let [bridge-map (reduce (fn [bs b]
+                             (assoc bs (:bridge b) b))
                            {} bridges)
         ;; Add bridges specified in links only
         all-bridges (reduce (fn [bs b]
@@ -181,21 +209,33 @@ General Options:
                             bridge-map
                             (keep :bridge links))
         ;; Enrich each bridge
-        bridges (reduce (fn [bs [k v]] (assoc bs k (enrich-bridge v)))
-                        {} all-bridges)
-        links (mapv #(enrich-link % bridges) links)
-        cfg (merge cfg {:links links
-                        :bridges bridges
-                        :containers {}
-                        :services {}})
+        enriched-bridges (reduce (fn [bs [k v]]
+                                   (assoc bs k (enrich-bridge v opts)))
+                                 {} all-bridges)
+        ;; Restructure links into a map to merge
+        link-map (reduce (fn [ls link]
+                           ;; Merge key is server/container + dev
+                           (let [lid (str (or (:service link)
+                                              (:container link))
+                                          ":" (get link :dev "eth0"))]
+                             (update ls lid deep-merge link)))
+                         {} links)
+        enriched-links (map #(enrich-link % enriched-bridges opts)
+                            (vals link-map))
+
+        cfg {:bridges enriched-bridges
+             :tunnels tunnels
+             :containers {}
+             :services {}}
         rfn (fn [kind cfg {:as x :keys [container service]}]
               (cond-> cfg
                 container (update-in [:containers container kind] conjv x)
                 service   (update-in [:services   service kind] conjv x)))
-        cfg (reduce (partial rfn :links) cfg links)
+        cfg (reduce (partial rfn :links) cfg enriched-links)
         cfg (reduce (partial rfn :commands) cfg commands)]
     cfg))
 
+
 (defn ajv-error-to-str [error]
   (let [path (:instancePath error)
         params (dissoc (:params error) :type :pattern :missingProperty)]
@@ -204,8 +244,7 @@ General Options:
          (if (not (empty? params)) (str " " params) ""))))
 
 (defn check-schema [data schema verbose]
-  (let [{:keys [info warn]} @ctx
-        ajv (Ajv. #js {:allErrors true})
+  (let [ajv (Ajv. #js {:allErrors true})
         validator (.compile ajv (->js schema))
         valid (validator (->js data))]
     (if valid
@@ -272,7 +311,6 @@ General Options:
   container properties from an event and the current pid of the
   network container. Updates iterable properties of the link
   (via link-add-offset) and adds the following keys:
-  - :container - the container properties (passed in)
   - :outer-pid - PID of the network namespace (passed in)
   - :pid - PID of this container
   - :dev-id - container name + container interface name
@@ -288,8 +326,7 @@ General Options:
         link (if (and outer-pid (not (:outer-dev link)))
                (assoc link :outer-dev (link-outer-dev link id index))
                link)
-        link (merge link {:container container
-                          :dev-id dev-id
+        link (merge link {:dev-id dev-id
                           :pid pid
                           :outer-pid outer-pid})]
     link))
@@ -350,6 +387,19 @@ General Options:
           res (run cmd {:quiet true})]
     (= 0 (:code res))))
 
+(defn intf-ipv4-addresses
+  "Return a sequence of IPv4 addresses for the interface."
+  [intf]
+  (P/let [cmd (str "ip -json addr show dev " intf)
+          res (run cmd {:quiet true})
+          addrs (when (= 0 (:code res))
+                  (js->clj (js/JSON.parse (:stdout res))
+                           :keywordize-keys true))]
+    (->> addrs
+         (mapcat :addr_info)
+         (filter #(= "inet" (:family %)))
+         (map :local))))
+
 (defn rename-docker-eth0
   "Rename docker's provided eth0 to DOCKER-INTF to prevent 'RTNETLINK
   answers: File exists' errors during creation of links that use
@@ -411,7 +461,8 @@ General Options:
     (if (not cmd)
       (info (str "Ignoring bridge/switch " bridge " for mode " mode))
       (P/let [_ (info "Creating bridge/switch" bridge)
-              res (run* [cmd (str "ip link set " bridge " up")])]
+              res (run* [cmd (str "ip link set " bridge " up")]
+                        {:id "bridge-create"})]
         (if (not= 0 (:code res))
           (error (str "Unable to create bridge/switch " bridge))
           (swap! ctx assoc-in [:network-state :bridges bridge :status] :created))
@@ -510,9 +561,10 @@ General Options:
                    (when outer-pid (str " --pid1 " outer-pid))
                    (when outer-dev (str " --intf1 " outer-dev))
                    (S/join ""
-                           (for [o LINK-ADD-OPTS]
-                             (when-let [v (get link o)]
-                               (str " --" (name o) " '" v "'")))))
+                           (for [o LINK-ADD-OPTS
+                                 :let [v (get link o [])]
+                                 vo (if (sequential? v) v [v])]
+                             (str " --" (name o) " '" vo "'"))))
           res (run cmd {:id dev-id})]
     (when (not= 0 (:code res))
       (error (str "Unable to add " (name type) " " dev-id)))
@@ -540,7 +592,7 @@ General Options:
   forwards defined by :forward property of 'link'."
   [link action]
   (P/let [{:keys [error]} @ctx
-          {:keys [outer-dev dev-id bridge ip forward]} link]
+          {:keys [dev-id bridge ip forward]} link]
     (P/all (for [fwd forward]
              (P/let [[port_a port_b proto] fwd
                      ip (S/replace ip #"/.*" "")
@@ -562,8 +614,9 @@ General Options:
   running in a container)"
   []
   (P/let [[cgroup mountinfo]
-          , (P/all [(read-file "/proc/self/cgroup" "utf8")
-                    (read-file "/proc/self/mountinfo" "utf8")])
+          , (P/catch (P/all [(read-file "/proc/self/cgroup" "utf8")
+                             (read-file "/proc/self/mountinfo" "utf8")])
+              #(vector "" ""))
           ;; docker
           d-cgroups (map second (re-seq #"/docker/([^/\n]*)" cgroup))
           ;; podman (root)
@@ -604,6 +657,18 @@ General Options:
                         (S/replace #"\." "-")))
            v])))
 
+(defn query-container-data
+  [container-obj]
+  (P/let
+    [container (inspect-container container-obj)
+     clabels (get-compose-labels container)
+     svc-num (:container-number clabels)]
+    {:name (->> container :Name (re-seq #"(.*/)?(.*)") first last)
+     :index (if svc-num (js/parseInt svc-num) 1)
+     :service (:service clabels)
+     :pid (-> container :State :Pid)
+     :labels clabels}))
+
 ;;;
 
 (defn docker-client
@@ -659,7 +724,7 @@ General Options:
     (condp = action
       "start"
       (if link-status
-        (error (str "Link " dev-id " already exists"))
+        (error (str "Link " dev-id " already in state: " link-status))
         (P/do
           (swap! ctx assoc-in status-path :creating)
           (link-add link)
@@ -730,29 +795,26 @@ General Options:
   if all containers/services are connected."
   [client {:keys [status id]}]
   (P/let
-    [{:keys [log info network-config compose-opts self-pid]} @ctx
+    [{:keys [log info network-config network-state compose-opts self-pid]} @ctx
      container-obj (get-container client id)
-     container (inspect-container container-obj)
-     cname (->> container :Name (re-seq #"(.*/)?(.*)") first last)
-     pid (-> container :State :Pid)
-
-     clabels (get-compose-labels container)
-     svc-name (:service clabels)
-     svc-num (:container-number clabels)
-     cindex (if svc-num (js/parseInt svc-num) 1)
-     container-info {:id id
-                     :name cname
-                     :index cindex
-                     :service svc-name
-                     :pid pid
-                     :labels clabels}
+     container-data (if (= "die" status)
+                      (P/let [ci (get-in network-state [:containers id])]
+                        (swap! ctx update-in [:network-state :containers]
+                               dissoc id)
+                        ci)
+                      (P/let [ci (query-container-data container-obj)]
+                        (swap! ctx update-in [:network-state :containers]
+                               assoc id ci)
+                        ci))
+     {cname :name clabels :labels} container-data
 
      svc-match? (and (let [p (:project compose-opts)]
                        (or (not p) (= p (:project clabels))))
                      (let [d (:project-working_dir compose-opts)]
                        (or (not d) (= d (:project-working_dir clabels)))))
      containers (get-in network-config [:containers cname])
-     services (when svc-match? (get-in network-config [:services svc-name]))
+     services (when svc-match?
+                (get-in network-config [:services (:service clabels)]))
      links (concat (:links containers) (:links services))
      commands (concat (:commands containers) (:commands services))]
     (if (and (not (seq links)) (not (seq commands)))
@@ -761,7 +823,7 @@ General Options:
         (info "Event:" status cname id)
         (P/all (for [link links
                      :let [link (link-instance-enrich
-                                  link container-info self-pid)]]
+                                  link container-data self-pid)]]
                  (modify-link link status)))
         (when (= "start" status)
           (P/all (for [{:keys [command]} commands]
@@ -838,15 +900,15 @@ General Options:
      kmod-ovs? (kmod-loaded? "openvswitch")
      kmod-mirred? (kmod-loaded? "act_mirred")
      docker-eth0? (and self-cid (intf-exists? "eth0"))
-     _ (swap! ctx merge {:default-bridge-mode (:default-bridge-mode opts)
-                         :default-mtu (:default-mtu opts)
-                         :kmod-ovs? kmod-ovs?
+     docker-eth0-addresses (when docker-eth0? (intf-ipv4-addresses "eth0"))
+     _ (swap! ctx merge {:kmod-ovs? kmod-ovs?
                          :kmod-mirred? kmod-mirred?
-                         :docker-eth0? docker-eth0?})
+                         :docker-eth0? docker-eth0?
+                         :docker-eth0-address (first docker-eth0-addresses)})
      network-config (P/-> (load-configs compose-file network-file)
                           (interpolate-walk env)
                           (check-schema schema verbose)
-                          (enrich-network-config))
+                          (enrich-network-config opts))
      _ (when show-config
          (println (js/JSON.stringify (->js network-config)))
          (js/process.exit 0))

package/src/conlink/util.cljs

@@ -44,7 +44,7 @@
   (js/process.exit code))
 
 (defn deep-merge [a b]
-  (merge-with #(cond (map? %1) (recur %1 %2)
+  (merge-with #(cond (map? %1) (deep-merge %1 %2)
                      (vector? %1) (vec (concat %1 %2))
                      (sequential? %1) (concat %1 %2)
                      :else %2)

package/test/test1.yaml

@@ -0,0 +1,26 @@
+name: "test1: compose file with embedded network config"
+
+env:
+  DC: "${{ process.env.DOCKER_COMPOSE || 'docker compose' }}"
+  COMPOSE_FILE: examples/test1-compose.yaml
+
+tests:
+  test1:
+    name: "compose file with embedded network config"
+    steps:
+      - exec: :host
+        run: |
+          ${DC} down --remove-orphans --volumes -t1
+          ${DC} up -d --force-recreate
+      - exec: :host
+        run: |
+          echo "waiting for conlink startup"
+          ${DC} logs network | grep "All links connected"
+        repeat: { retries: 30, interval: '1s' }
+
+      - {exec: h1, run: ping -c1 -w2 10.0.0.100}
+      - {exec: h2, run: ping -c1 -w2 192.168.1.100}
+      - {exec: h3, run: ping -c1 -w2 172.16.0.100}
+
+      - exec: :host
+        run: ${DC} down --remove-orphans --volumes -t1

package/test/test10.yaml

@@ -0,0 +1,46 @@
+name: "test10: port forwarding" 
+
+env:
+  DC: "${{ process.env.DOCKER_COMPOSE || 'docker compose' }}"
+  COMPOSE_FILE: examples/test10-compose.yaml
+
+tests:
+  test10:
+    name: "port forwarding"
+    steps:
+      - exec: :host
+        run: |
+          ${DC} down --remove-orphans --volumes -t1
+          ${DC} up -d --force-recreate
+      - exec: :host
+        run: |
+          echo "waiting for conlink startup"
+          ${DC} logs network | grep "All links connected"
+        repeat: { retries: 30, interval: '1s' }
+      - exec: node1
+        run: ip addr | grep "10\.1\.0\.1"
+        repeat: { retries: 10, interval: '2s' }
+
+      # Check ping between replicas
+      - {exec: node2, run: ping -c1 -w2 10.2.0.2}
+      # Check ping across router
+      - {exec: node1, run: ping -c1 -w2 10.2.0.1}
+      - {exec: node1, run: ping -c1 -w2 10.2.0.2}
+      - {exec: node2, run: ping -c1 -w2 10.1.0.1}
+      # Check ping across router
+      - exec: :host
+        run: 'curl -sS "http://0.0.0.0:3080" | grep "log"'
+        repeat: { retries: 10, interval: '2s' }
+      - exec: :host
+        run: 'curl -sS "http://0.0.0.0:8080" | grep "log"'
+        repeat: { retries: 10, interval: '2s' }
+      - exec: :host
+        run: 'curl -sS "http://0.0.0.0:80" | grep "share"'
+        repeat: { retries: 10, interval: '2s' }
+      - exec: :host
+        run: 'curl -sS "http://0.0.0.0:81" | grep "share"'
+        repeat: { retries: 10, interval: '2s' }
+
+      - exec: :host
+        run: ${DC} down --remove-orphans --volumes -t1
+

package/test/test2.yaml

@@ -0,0 +1,37 @@
+name: "test2: separate config and scaling" 
+
+env:
+  DC: "${{ process.env.DOCKER_COMPOSE || 'docker compose' }}"
+  COMPOSE_FILE: examples/test2-compose.yaml
+
+tests:
+  test2:
+    name: "separate config and scaling" 
+    steps:
+      - exec: :host
+        run: |
+          ${DC} down --remove-orphans --volumes -t1
+          ${DC} up -d --force-recreate
+      - exec: :host
+        run: |
+          echo "waiting for conlink startup"
+          ${DC} logs network | grep "All links connected"
+        repeat: { retries: 30, interval: '1s' }
+      - {exec: node, index: 1, run: ping -c1 -w2 10.0.1.2}
+      - {exec: node, index: 2, run: ping -c1 -w2 10.0.1.1}
+      - {exec: node, index: 1, run: ping -c1 -w2 8.8.8.8}
+      - {exec: node, index: 2, run: ping -c1 -w2 8.8.8.8}
+
+      - exec: :host
+        run: |
+          echo "Scale the nodes from 2 to 5"
+          ${DC} up -d --scale node=5
+      - exec: node
+        index: 5
+        run: ip addr | grep "10\.0\.1\.5"
+        repeat: { retries: 10, interval: '2s' }
+      - {exec: node, index: 2, run: ping -c1 -w2 10.0.1.5}
+      - {exec: node, index: 5, run: ping -c1 -w2 8.8.8.8}
+
+      - exec: :host
+        run: ${DC} down --remove-orphans --volumes -t1

package/test/test4.yaml

@@ -0,0 +1,80 @@
+name: "test4: multiple compose and mdc" 
+
+env:
+  DC: "${{ process.env.DOCKER_COMPOSE || 'docker compose' }}"
+  MODES_DIR: examples/test4-multiple/modes
+
+tests:
+  node1:
+    name: "mdc node1"
+    steps:
+      - exec: :host
+        run: |
+          ./mdc node1
+          ${DC} down --remove-orphans --volumes -t1
+          ${DC} up -d --force-recreate
+      - exec: :host
+        run: |
+          echo "waiting for conlink startup"
+          ${DC} logs network | grep "All links connected"
+        repeat: { retries: 30, interval: '1s' }
+      - exec: r0
+        run: ip addr | grep "10\.1\.0\.100"
+        repeat: { retries: 10, interval: '2s' }
+
+      # Ping the r0 router host from node1
+      - {exec: node1, index: 1, run: ping -c1 -w2 10.0.0.100}
+
+      - exec: :host
+        run: ${DC} down --remove-orphans --volumes -t1
+
+  node1-nodes2:
+    name: "mdc node1,nodes2"
+    steps:
+      - exec: :host
+        run: |
+          ./mdc node1,nodes2
+          ${DC} down --remove-orphans --volumes -t1
+          ${DC} up -d --force-recreate
+      - exec: :host
+        run: |
+          echo "waiting for conlink startup"
+          ${DC} logs network | grep "All links connected"
+        repeat: { retries: 30, interval: '1s' }
+      - exec: node2
+        index: 2
+        run: ip addr | grep "10\.2\.0\.2"
+        repeat: { retries: 10, interval: '2s' }
+
+      # From both node2 replicas, ping node1 across the r0 router
+      - {exec: node2, index: 1, run: ping -c1 -w2 10.1.0.1}
+      - {exec: node2, index: 2, run: ping -c1 -w2 10.1.0.1}
+      # From node1, ping both node2 replicas across the r0 router
+      - {exec: node1, index: 1, run: ping -c1 -w2 10.2.0.1}
+      - {exec: node1, index: 1, run: ping -c1 -w2 10.2.0.2}
+
+      - exec: :host
+        run: ${DC} down --remove-orphans --volumes -t1
+
+  all:
+    name: "mdc all"
+    steps:
+      - exec: :host
+        run: |
+          ./mdc all
+          ${DC} down --remove-orphans --volumes -t1
+          ${DC} up -d --force-recreate
+      - exec: :host
+        run: |
+          echo "waiting for conlink startup"
+          ${DC} logs network | grep "All links connected"
+        repeat: { retries: 30, interval: '1s' }
+      - exec: r0
+        run: /scripts/wait.sh -t 10.0.0.100:80
+
+      # From node2, download from the web server in r0
+      - {exec: node2, index: 1, run: wget -O- 10.0.0.100}
+      - {exec: node2, index: 2, run: wget -O- 10.0.0.100}
+
+      - exec: :host
+        run: ${DC} down --remove-orphans --volumes -t1