conlink 2.0.1 → 2.0.2

package/README.md

@@ -258,8 +258,7 @@ container (`node1`) and switch (`s1`) that is connected to the router
 defined in the first compose file.
 
 ```
-echo "COMPOSE_FILE=examples/test4-multiple/base-compose.yaml:examples/test4-multiple/node1-compose.yaml" > .env
-docker-compose up --build --force-recreate
+MODES_DIR=./examples/test4-multiple/modes ./mdc node1 up --build --force-recreate
 ```
 
 Ping the router host from `node`:
@@ -273,8 +272,7 @@ two node2 replicas and a switch (`s2`) that is connected to the
 router.
 
 ```
-echo "COMPOSE_FILE=examples/test4-multiple/base-compose.yaml:examples/test4-multiple/node1-compose.yaml:examples/test4-multiple/nodes2-compose.yaml" > .env
-docker-compose up --build --force-recreate
+MODES_DIR=./examples/test4-multiple/modes ./mdc node1,nodes2 up --build --force-recreate
 ```
 
 From both `node2` replicas, ping `node1` across the switches and `r0` router:
@@ -289,8 +287,7 @@ conlink using an addition network file `web-network.yaml`. The network
 file starts up a simple web server on the router.
 
 ```
-echo "COMPOSE_FILE=examples/test4-multiple/base-compose.yaml:examples/test4-multiple/node1-compose.yaml:examples/test4-multiple/nodes2-compose.yaml:examples/test4-multiple/all-compose.yaml" > .env
-docker-compose up --build --force-recreate
+MODES_DIR=./examples/test4-multiple/modes ./mdc node1,nodes2,web up --build --force-recreate
 ```
 
 From the second `node2`, perform a download from the web server running on the
@@ -300,6 +297,14 @@ router host:
 docker-compose exec --index 2 node2 wget -O- 10.0.0.100
 ```
 
+We can simplify the above launch by using the `all` mode, which contains
+depends on `node1`, `nodes2`, and `web`.  Each of those modes depends on
+`base`, so there's no need to specify that again (transitive deps).
+
+```
+MODES_DIR=./examples/test4-multiple/modes ./mdc all up --build --force-recreate
+```
+
 Remove the `.env` file as a final cleanup step:
 
 ```

package/examples/test4-multiple/modes/all/deps

@@ -0,0 +1 @@
+node1 nodes2 web

package/examples/test4-multiple/{base-compose.yaml → modes/base/compose.yaml}

@@ -10,7 +10,7 @@ services:
         - {bridge: s0, dev: e0, ip: "10.0.0.100/24"}
 
   network:
-    build: {context: ../..}
+    build: {context: .}
     image: conlink
     pid: host
     network_mode: none
@@ -19,7 +19,7 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - /var/lib/docker:/var/lib/docker
-      - ../../:/test
+      - ./:/test
     working_dir: /test
-    command: /app/build/conlink.js --compose-file ${COMPOSE_FILE:-examples/test4-multiple/base-compose.yaml}
+    command: /app/build/conlink.js --compose-file ${COMPOSE_FILE:?COMPOSE_FILE must be set}
 

package/examples/test4-multiple/{node1-compose.yaml → modes/node1/compose.yaml}

@@ -5,9 +5,6 @@ x-network:
     - {service: r0, bridge: s1, dev: e1, ip: "10.1.0.100/24"}
 
 services:
-  network:
-    command: /app/build/conlink.js --compose-file ${COMPOSE_FILE:-examples/test4-multiple/node1-compose.yaml}
-
   node1:
     image: alpine
     network_mode: none

package/examples/test4-multiple/{nodes2-compose.yaml → modes/nodes2/compose.yaml}

@@ -4,11 +4,7 @@ x-network:
   links:
     - {service: r0, bridge: s2, dev: e2, ip: "10.2.0.100/24"}
 
-
 services:
-  network:
-    command: /app/build/conlink.js --compose-file ${COMPOSE_FILE:-examples/test4-multiple/nodes2-compose.yaml}
-
   node2:
     image: alpine
     scale: 2
@@ -17,4 +13,3 @@ services:
     x-network:
       links:
         - {bridge: s2, ip: "10.2.0.1/16", route: "default via 10.2.0.100"}
-

package/examples/test4-multiple/modes/web/compose.yaml

@@ -0,0 +1,5 @@
+version: "2.4"
+
+x-network:
+  commands:
+    - {service: r0, command: "python3 -m http.server 80"}

package/examples/test4-multiple/modes/web/deps

@@ -0,0 +1 @@
+base

package/mdc

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+set -e
+shopt -s dotglob  # recursive copy of dot files too
+
+die() { echo >&2 "${*}"; exit 1; }
+vecho() { [ "${VERBOSE}" ] && echo >&2 "${*}" || true; }
+realpath() { [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"; }
+
+[ "${*}" ] || die "Usage: ${0} MODES [DOCKER-COMPOSE-ARGS]"
+
+VERBOSE="${VERBOSE:-}"
+MODES_DIR="${MODES_DIR:-./modes}"
+ENV_FILE="${ENV_FILE:-.env}"
+MDC_FILES_DIR="${MDC_FILES_DIR:-./.files}"
+LS=$(which ls)
+RESOLVE_DEPS="${RESOLVE_DEPS-./node_modules/@lonocloud/resolve-deps/resolve-deps.py}"
+DOCKER_COMPOSE="${DOCKER_COMPOSE:-docker-compose}"
+
+MODE_SPEC="${1}"; shift
+if [ "${RESOLVE_DEPS}" ]; then
+    MODES="$(${RESOLVE_DEPS} "${MODES_DIR}" ${MODE_SPEC})"
+else
+    MODES="${MODE_SPEC//,/ }"
+fi
+
+echo >&2 "MODES:    ${MODES}"
+vecho "ENV_FILE: ${ENV_FILE}"
+
+declare -A FINISHED
+COMPOSE_FILE=./.compose-empty.yaml
+COMPOSE_PROFILES=
+MDC_MODE_DIRS=
+cat /dev/null > ${ENV_FILE}-mdc-tmp
+echo -e "version: '2.4'\nservices: {}" > ./.compose-empty.yaml
+
+vecho "Removing ${MDC_FILES_DIR}"
+case "$(basename ${MDC_FILES_DIR})" in
+    .|/) die "MDC_FILES_DIR must not be '.' or '/'";;
+esac
+[ -d "${MDC_FILES_DIR}" ] && rm -r${VERBOSE:+v} ${MDC_FILES_DIR}/
+mkdir -p "${MDC_FILES_DIR}"
+
+for mode in ${MODES}; do
+    # Only process each mode once
+    [ "${FINISHED[${mode}]}" ] && continue
+    FINISHED["${mode}"]=1
+
+    # mode dir must exist
+    [ -d "${MODES_DIR}/${mode}" ] || \
+        die "No mode dir found for ${mode}"
+
+    MDC_MODE_DIRS="${MDC_MODE_DIRS},${MODES_DIR}/${mode}"
+
+    # mode can refer to a compose file in multiple ways
+    cfiles="${MODES_DIR}/${mode}/compose.yaml ${MODES_DIR}/${mode}/compose.yml ${MODES_DIR}/${mode}/docker-compose.yaml ${MODES_DIR}/${mode}/docker-compose.yml"
+    for cfile in ${cfiles}; do
+        if [ -e "${cfile}" ]; then
+            COMPOSE_FILE="${COMPOSE_FILE}:${cfile}"
+            break
+        fi
+    done
+
+    # Add MODE_ prefixed compose profile for each mode
+    COMPOSE_PROFILES="${COMPOSE_PROFILES},MODE_${mode}"
+
+    # if there is a mode specific env file then include it
+    efiles="${MODES_DIR}/${mode}/env ${MODES_DIR}/${mode}/.env"
+    for efile in ${efiles}; do
+        if [ -e "${efile}" ]; then
+            echo "### mdc begin mode ${mode}" >> ${ENV_FILE}-mdc-tmp
+            vecho "cat ${efile} >> ${ENV_FILE}-mdc-tmp"
+            cat ${efile} >> ${ENV_FILE}-mdc-tmp
+            echo "### mdc end mode ${mode}" >> ${ENV_FILE}-mdc-tmp
+        fi
+    done
+
+    # if there are mode specific files then copy them to MDC_FILES_DIR
+    if [ -d "${MODES_DIR}/${mode}" ]; then
+        for vfd in $(cd ${MODES_DIR}/${mode} && $LS -d */files 2>/dev/null || true); do
+            dest=${MDC_FILES_DIR}/${vfd%/files}
+            mkdir -p ${dest}
+            vecho cp -a ${MODES_DIR}/${mode}/${vfd}/* ${dest}
+            cp -a ${MODES_DIR}/${mode}/${vfd}/* ${dest}
+        done
+    fi
+done
+
+COMPOSE_FILE="${COMPOSE_FILE#:}"
+vecho "COMPOSE_FILE: ${COMPOSE_FILE}"
+echo "COMPOSE_FILE=${COMPOSE_FILE}" >> ${ENV_FILE}-mdc-tmp
+echo "COMPOSE_DIR=$(realpath $(dirname ${ENV_FILE}))" >> ${ENV_FILE}-mdc-tmp
+
+COMPOSE_PROFILES="${COMPOSE_PROFILES#,}"
+vecho "COMPOSE_PROFILES: ${COMPOSE_PROFILES}"
+echo "COMPOSE_PROFILES=${COMPOSE_PROFILES}" >> ${ENV_FILE}-mdc-tmp
+
+MDC_MODE_DIRS="${MDC_MODE_DIRS#,}"
+vecho "MDC_MODE_DIRS: ${MDC_MODE_DIRS}"
+echo "MDC_MODE_DIRS=\"${MDC_MODE_DIRS}\"" >> ${ENV_FILE}-mdc-tmp
+
+vecho mv ${ENV_FILE}-mdc-tmp ${ENV_FILE}
+mv ${ENV_FILE}-mdc-tmp ${ENV_FILE}
+
+if [ "${*}" ]; then
+    vecho "Running: ${DOCKER_COMPOSE} --env-file "${ENV_FILE}" ${@}"
+    exec ${DOCKER_COMPOSE} --env-file "${ENV_FILE}" "${@}"
+fi

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "conlink",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "description": "conlink -  Declarative Low-Level Networking for Containers",
   "repository": "https://github.com/LonoCloud/conlink",
   "license": "SEE LICENSE IN LICENSE",
   "dependencies": {
+    "@lonocloud/resolve-deps": "^0.0.2",
     "ajv": "^8.12.0",
     "dockerode": "^3.3.4",
     "nbb": "^1.2.179",

package/scripts/copy.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+# Copyright (c) 2023, Viasat, Inc
+# Licensed under MPL 2.0
+
+# Add files from SRC_DIR to DST_DIR with string interpolation.
+# Any '{{FOO}}' tokens are replaced with value of corresponding
+# environment variable FOO (but only if defined).
+
+die() { local ret=${1}; shift; echo >&2 "${*}"; exit $ret; }
+
+case "${1}" in -T|--template) TEMPLATE=1; shift ;; esac
+
+src_dir="${1}"; shift || die 2 "Usage: ${0} [-T|--template] SRC_DIR DST_DIR"
+dst_dir="${1}"; shift || die 2 "Usage: ${0} [-T|--template] SRC_DIR DST_DIR"
+[ "${1}" = "--" ] && shift
+
+[ -d "${src_dir}" ] || die 2 "Not a directory: '${src_dir}'"
+[ -d "${dst_dir}" ] || die 2 "Not a directory: '${dst_dir}'"
+
+(cd "${src_dir}" && find . -type f) | while read src_file; do
+  src="${src_dir}/${src_file}"
+  dst="${dst_dir}/${src_file}"
+  mkdir -p $(dirname "${dst}") || die 1 "Failed to make target directory"
+  echo cp -a "${src}" "${dst}"
+  cp -a "${src}" "${dst}" || die 1 "Failed to copy file"
+  # TODO: make this configurable
+  chown root.root "${dst}" || die 1 "Unable to set ownership"
+
+  [ -z "${TEMPLATE}" ] && continue
+
+  # match all {{FOO}} style variables and replace from environment
+  for v in $(cat "${dst}" | grep -o '{{[^ }{]*}}' | sed 's/[}{]//g' | sort -u); do
+    if set | grep -qs "^${v}="; then
+      val=$(set | grep "^${v}=" | cut -f 2 -d '=' \
+          | sed "s/^['\"]\(.*\)['\"]$/\1/" \
+          | sed 's/[\/&]/\\&/g')
+      echo "Replacing '{{${v}}}' with '${val}' in '${dst}'"
+      sed -i "s/{{${v}}}/${val}/g" "${dst}"
+    fi
+  done
+done
+
+if [ "${*}" ]; then
+    exec "${@}"
+else
+    true
+fi

package/scripts/wait.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+
+# Copyright (c) 2023, Viasat, Inc
+# Licensed under MPL 2.0
+
+die() { local ret=${1}; shift; echo >&2 "${*}"; exit $ret; }
+
+NC=$(command -v nc 2>/dev/null)
+SOCAT=$(command -v socat 2>/dev/null)
+BASH=$(command -v bash 2>/dev/null)
+WAIT_SLEEP=${WAIT_SLEEP:-1}
+
+do_sleep() {
+  echo "Failed: '${typ} ${arg}'. Sleep ${WAIT_SLEEP} seconds before retry"
+  sleep ${WAIT_SLEEP}
+}
+check_tcp() {
+  if   [ "${NC}" ];    then ${NC} -z -w 1 ${1} ${2} > /dev/null
+  elif [ "${SOCAT}" ]; then ${SOCAT} /dev/null TCP:${1}:${2},connect-timeout=2
+  elif [ "${BASH}" ];  then timeout 1 ${BASH} -c "echo > /dev/tcp/${1}/${2}"
+  else die 1 "Could not find nc, socat, or bash"
+  fi
+}
+
+while [ "${*}" ]; do
+  typ="${1}"; shift
+  arg="${1}"
+  [ "${arg}" = "--" ] && die 2 "No arg found for type '${typ}'"
+  case "${typ}" in
+    --) break; ;;
+    -f|--file)
+      while [ ! -e "${arg}" ]; do do_sleep; done
+      echo "File '${arg}' exists"
+      ;;
+    -i|--if|--intf)
+      while [ ! -e /sys/class/net/${arg}/ifindex ]; do do_sleep; done
+      echo "Interface '${arg}' exists"
+      ;;
+    -I|--ip)
+      while ! grep -qs "^${arg}\>" /proc/net/route; do do_sleep; done
+      echo "Interface '${arg}' has IP/routing"
+      ;;
+    -t|--tcp)
+      host=${arg%:*}
+      port=${arg##*:}
+      [ "${host}" -a "${port}" ] || die 2 "Illegal host/port '${arg}'"
+      while ! check_tcp ${host} ${port}; do do_sleep; done
+      echo "TCP listener is reachable at '${arg}' "
+      ;;
+    -u|--umask)
+      umask ${arg}
+      echo "Set umask to ${arg}"
+      ;;
+    -c|--cmd|--command)
+      while ! ${arg}; do do_sleep; done
+      echo "Command successful: ${arg}"
+      ;;
+    -s|--sleep)
+      WAIT_SLEEP=${arg}
+      echo "Changed WAIT_SLEEP from ${WAIT_SLEEP} to ${arg}"
+      ;;
+    *)
+      echo "Unknown option: ${typ}"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if [ "${*}" ]; then
+  echo "Running: ${*}"
+  exec ${*}
+fi

package/.env

@@ -1,3 +0,0 @@
-HOST_INTERFACE=eni0
-NODE1_HOST_ADDRESS=192.168.0.32
-NODE2_HOST_ADDRESS=192.168.0.33

package/TODO

@@ -1,34 +0,0 @@
-- MVP for ViaBox:
-    - [x] compose/x-network file loading
-    - [x] multiple config sources and merging
-    - [x] link route config
-    - [x] filtering on project and workdir
-    - [x] interface and MAC iteration
-    - [x] variable templating
-    - [x] *vlan type interfaces
-
-- Near term:
-    - [x] dummy interfaces
-    - [x] arbitrary container commands
-    - [x] schema validation
-    - [x] code comments/documentation
-    - [x] tunnel interfaces
-    - [x] tc/qdisc settings
-    - [x] fix/test all examples (6 and 9 remaining)
-    - [x] add net2dot
-    - [ ] add outer-netem (and match all link-add params to link keys)
-
-- Further term:
-    - [ ] CNI networking support
-        - conlink runs in container listening for events on a UDS
-          (intead of docker events)
-        - an outer conlink command is the CNI client that formats
-          events to send over the UDS to the inner conlink
-    - [ ] multiple routes
-    - [ ] ovs flow config
-    - [ ] Multiple bridge-modes
-        - bridge-mode as part of the domain definition so that the
-          same conlink instances can support multiple bridge modes
-          simultaneously (with a default for links that don't
-          specify).
-