@webship.co/workspace 1.0.0-beta1 → 1.0.0-beta2

package/core/scripts/functions/fun-project-name.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# The one place a project name gets cleaned up, called from every cmd-*.sh that takes one.
+#
+# A directory name copy-pasted or tab-completed carries a trailing slash more often than not —
+# `cd dev/v11x00b1<TAB>` appends one, and so does copying a name straight out of a directory
+# listing. That slash is never part of the project name, only of how it arrived, so every command
+# that takes PROJECT_NAME as its positional trims it before using it anywhere.
+#
+# One real implementation rather than the same trim (and the same explanation) pasted into 42
+# files: each command still calls it explicitly, right after its own `shift $#;`, because
+# PROJECT_NAME does not exist until that command's own argparse has run — bootstrap.sh loads this
+# function before that, but cannot act on a variable nothing has set yet.
+function trim_project_name() {
+  if [ "$#" -gt 0 ] ; then
+    printf '%s' "${1%/}" ;
+  else
+    PROJECT_NAME="${PROJECT_NAME%/}" ;
+  fi
+}

package/core/scripts/functions/fun-rag.sh

@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+
+# The rag workspace: the Milvus vector database that holds every project's RAG index, and the
+# commands that read it.
+#
+# The split mirrors graphs/: cmd-tools-ragify.sh (in each project's own workspace folder) BUILDS
+# an index, and this command is what you do with it afterwards — list, inspect, search, drop, and
+# hand it to the Claude Code CLI over MCP.
+#
+# Unlike a graph, an index is not a file: it is rows inside Milvus. So there is nothing to move
+# out of the project and nothing to keep in the folder — `rag/` holds the database instance
+# itself, and the collections live in it.
+
+# pymilvus 2.6.x is the client for a Milvus 2.6.x server. Pinned below 3: pymilvus 3.x tracks the
+# Milvus 3.0 line released on 2026-07-29, and pairing a 3.x client with the 2.6 server this
+# workspace installs by default is exactly the combination that fails at connect time rather than
+# at install time.
+RAG_PYMILVUS="pymilvus>=2.6,<3";
+RAG_OPENAI="openai>=1.40";
+
+# Run the Python worker (core/scripts/libs/rag_index.py).
+#
+# Through `uv run --with`, not an installed tool: the dependency set is resolved and cached by uv,
+# so there is no venv of ours to create, no PATH entry to add and nothing to keep in sync with the
+# rest of the workspace. The OpenAI client is only pulled in when a run actually asks for dense
+# embeddings.
+function rag_python() {
+  local extra="";
+  case " $* " in
+    *" --embedding openai "*|*" --openai-key "*) extra="--with ${RAG_OPENAI}" ;;
+  esac
+  # shellcheck disable=SC2086
+  uv run --quiet --with "${RAG_PYMILVUS}" ${extra} \
+    python3 "${WORKSPACE_SCRIPTS}/libs/rag_index.py" "$@" ;
+}
+
+# Which Milvus instance a project's index lives in.
+#
+# One shared instance is the default, and that is right for almost everything: a collection is
+# already an isolation boundary, and a second Milvus is another etcd, another MinIO and another
+# ~2 GB of containers. A project can still be BOUND to its own — a client engagement whose vectors
+# must be backed up and handed over on their own, or one that needs a different Milvus version.
+#
+# The binding lives here, not in the project, for the same reason a graph does not: half the
+# projects worth indexing are repositories we ship, and a repo must not gain a file because
+# somebody indexed it. One line per bound project; anything unbound simply is not in the file.
+RAG_BINDINGS_FILE="${WORKSPACE_ROOT}/rag/bindings.yml";
+
+function rag_binding_get() {
+  local workspace="$1" project="$2";
+  [ -f "${RAG_BINDINGS_FILE}" ] || return 0 ;
+  sed -n "s|^${workspace}/${project}:[[:space:]]*\([A-Za-z0-9_-]\+\)[[:space:]]*$|\1|p" "${RAG_BINDINGS_FILE}" | head -1 ;
+}
+
+function rag_binding_set() {
+  local workspace="$1" project="$2" instance="$3" tmp;
+  mkdir -p "$(dirname "${RAG_BINDINGS_FILE}")" ;
+  if [ ! -f "${RAG_BINDINGS_FILE}" ] ; then
+    cat > "${RAG_BINDINGS_FILE}" <<'HEADEREOF'
+# Which Milvus instance holds each project's RAG index.
+#
+# Written by cmd-tools-ragify.sh --milvus <instance> and by the dashboard's RAG menu. A project
+# that is not listed here uses milvus.project from core/config/workspace.rag.settings.yml.
+#
+# Machine-local state, like the graphs store: it describes what is built on THIS machine.
+HEADEREOF
+  fi
+  tmp="$(mktemp)";
+  grep -v "^${workspace}/${project}:" "${RAG_BINDINGS_FILE}" > "${tmp}" 2>/dev/null ;
+  if [ -n "${instance}" ] ; then
+    printf '%s/%s: %s\n' "${workspace}" "${project}" "${instance}" >> "${tmp}" ;
+  fi
+  mv "${tmp}" "${RAG_BINDINGS_FILE}" ;
+}
+
+# The instance a command should talk to: an explicit argument, then the binding, then the default.
+function rag_resolve_instance() {
+  local workspace="$1" project="$2" given="$3" bound;
+  if [ -n "${given}" ] && [ "${given}" != '_settings_' ] ; then
+    printf '%s' "${given}" ; return 0 ;
+  fi
+  bound="$(rag_binding_get "${workspace}" "${project}")";
+  if [ -n "${bound}" ] ; then printf '%s' "${bound}" ; return 0 ; fi
+  printf '%s' "$(milvus_resolve_project '_settings_')";
+}
+
+# Every Milvus instance built in the rag workspace: a directory with the compose file this
+# tooling writes. Used by `-a servers` and by the dashboard's server picker.
+function rag_instances() {
+  local dir;
+  for dir in "${WORKSPACE_ROOT}"/rag/*/ ; do
+    [ -f "${dir}.ddev/docker-compose.milvus.yaml" ] || continue ;
+    basename "${dir}" ;
+  done
+}
+
+# The collection name for one project: <prefix>_<workspace>_<project>, with everything Milvus
+# does not allow in an identifier folded to an underscore. Deterministic, so ragify writes and
+# this command reads the same name without keeping state anywhere.
+function rag_collection_name() {
+  local workspace="$1" project="$2" prefix;
+  prefix="$(milvus_setting "${rag_collection_prefix}")"; prefix="${prefix:-ws}";
+  printf '%s_%s_%s' "${prefix}" "${workspace}" "${project}" | tr -c 'A-Za-z0-9_\n' '_' ;
+}
+
+# Resolve what the user named to a collection that exists.
+#
+# Accepts <workspace>/<project>, a bare <project> (matched across workspaces), or the raw
+# collection name. Ambiguity is reported rather than resolved by sort order: drop is one of the
+# actions here and it does not ask twice.
+function rag_resolve_collection() {
+  local wanted="$1" uri="$2" token="$3" all matches;
+  all="$(rag_python --uri "${uri}" --token "${token}" --json collections 2>/dev/null \
+        | python3 -c "import json,sys
+try:
+    print('\n'.join(row['collection'] for row in json.load(sys.stdin)))
+except Exception:
+    pass")";
+  [ -n "${all}" ] || return 0 ;
+
+  case "${wanted}" in
+    */*) matches="$(printf '%s\n' "${all}" | grep -x "$(rag_collection_name "${wanted%%/*}" "${wanted##*/}")")" ;;
+    *)   matches="$(printf '%s\n' "${all}" | grep -x "${wanted}")";
+         [ -n "${matches}" ] || matches="$(printf '%s\n' "${all}" | grep -E "_${wanted}$")" ;;
+  esac
+  printf '%s' "${matches}";
+}
+
+# One resolved collection, or a sentence saying why not. Sets RAG_COLLECTION.
+function rag_require_collection() {
+  local uri="$1" token="$2" count;
+  if [ "${RAG_NAME}" == '_all_' ] ; then
+    echo "Name the index: bash cmd-tools-rag.sh <workspace>/<project> -a ${RAG_ACTION}";
+    echo "  (-a list shows them)";
+    return 1 ;
+  fi
+  RAG_COLLECTION="$(rag_resolve_collection "${RAG_NAME}" "${uri}" "${token}")";
+  count="$(printf '%s' "${RAG_COLLECTION}" | grep -c . )";
+  if [ "${count}" -eq 0 ] ; then
+    echo "No index matches '${RAG_NAME}'.";
+    echo "  Build it:  cd ${WORKSPACE_ROOT}/<workspace> && bash cmd-tools-ragify.sh ${RAG_NAME##*/}";
+    return 1 ;
+  fi
+  if [ "${count}" -gt 1 ] ; then
+    echo "'${RAG_NAME}' matches ${count} indexes — name the workspace too:";
+    printf '  %s\n' ${RAG_COLLECTION};
+    return 1 ;
+  fi
+  return 0 ;
+}
+
+function rag_command() {
+  local project uri token target_ws target_project;
+
+  # An index named as <workspace>/<project> resolves to the instance THAT project is bound to,
+  # which is the whole point of binding: `-a search demos/website_starter` must reach the server holding
+  # website_starter's vectors, not the default one.
+  case "${RAG_NAME}" in
+    */*) target_ws="${RAG_NAME%%/*}"; target_project="${RAG_NAME##*/}" ;;
+    *)   target_ws=""; target_project="" ;;
+  esac
+  if [ -n "${target_ws}" ] ; then
+    project="$(rag_resolve_instance "${target_ws}" "${target_project}" "${MILVUS_PROJECT}")";
+  else
+    project="$(milvus_resolve_project "${MILVUS_PROJECT}")";
+  fi
+  uri="$(milvus_uri "${project}")";
+  token="$(milvus_token)";
+
+  case "${RAG_ACTION}" in
+    list|serve|mcp-register|mcp-unregister|status|servers|bind) : ;;
+    *) milvus_require_running "${project}" || return 1 ;;
+  esac
+
+  case "${RAG_ACTION}" in
+    servers)
+      local instance bound_count;
+      printf '%-28s %-10s %-28s %s\n' "SERVER" "STATE" "URI" "BOUND PROJECTS";
+      while read -r instance ; do
+        [ -n "${instance}" ] || continue ;
+        # `grep -c` already prints 0 when it matches nothing — and exits 1 while doing so, so a
+        # `|| echo 0` fallback appended a SECOND line and the count became "0\n0", which printf
+        # then spread across two rows. The missing-file case is what the default is for.
+        bound_count="$(grep -c ": ${instance}$" "${RAG_BINDINGS_FILE}" 2>/dev/null)";
+        bound_count="${bound_count:-0}";
+        printf '%-28s %-10s %-28s %s\n' "${instance}" \
+          "$(docker ps --format '{{.Names}}' | grep -qx "ddev-${instance}-milvus-standalone" && echo running || echo stopped)" \
+          "$(milvus_uri "${instance}")" \
+          "${bound_count}$([ "${instance}" == "$(milvus_resolve_project '_settings_')" ] && echo ' (default)')";
+      done <<< "$(rag_instances)"
+      echo "";
+      echo "Build another:  bash cmd-milvus-project.sh <name>";
+      echo "Bind a project: bash cmd-tools-rag.sh <workspace>/<project> -a bind -m <name>";
+      ;;
+    bind)
+      if [ -z "${target_ws}" ] ; then
+        echo "Name the project as <workspace>/<project>: bash cmd-tools-rag.sh demos/website_starter -a bind -m <server>";
+        return 1 ;
+      fi
+      if [ "${MILVUS_PROJECT}" == '_settings_' ] ; then
+        echo "Name the server with -m: bash cmd-tools-rag.sh ${RAG_NAME} -a bind -m <server>";
+        echo "  (-a servers lists them; -m default unbinds and falls back to the default server)";
+        return 1 ;
+      fi
+      if [ "${MILVUS_PROJECT}" == 'default' ] ; then
+        rag_binding_set "${target_ws}" "${target_project}" "" ;
+        echo "${RAG_NAME} now uses the default server ($(milvus_resolve_project '_settings_')).";
+        return 0 ;
+      fi
+      if ! printf '%s\n' "$(rag_instances)" | grep -qx "${MILVUS_PROJECT}" ; then
+        echo "No Milvus instance called '${MILVUS_PROJECT}' in ${WORKSPACE_ROOT}/rag.";
+        echo "  Build it first: cd ${WORKSPACE_ROOT}/rag && bash cmd-milvus-project.sh ${MILVUS_PROJECT}";
+        return 1 ;
+      fi
+      rag_binding_set "${target_ws}" "${target_project}" "${MILVUS_PROJECT}" ;
+      echo "${RAG_NAME} is now indexed into ${MILVUS_PROJECT} ($(milvus_uri "${MILVUS_PROJECT}")).";
+      echo "  Re-index it there: cd ${WORKSPACE_ROOT}/${target_ws} && bash cmd-tools-ragify.sh ${target_project}";
+      echo "  The old collection stays where it was — drop it with -a remove before rebinding if it is stale.";
+      ;;
+    list)
+      milvus_require_running "${project}" || return 1 ;
+      echo "Milvus ${project} on ${uri}";
+      echo "";
+      rag_python --uri "${uri}" --token "${token}" collections ;
+      ;;
+    info)
+      rag_require_collection "${uri}" "${token}" || return 1 ;
+      rag_python --uri "${uri}" --token "${token}" info --collection "${RAG_COLLECTION}" ;
+      ;;
+    search)
+      if [ -z "${QUERY}" ] ; then
+        echo "Give a question: bash cmd-tools-rag.sh <workspace>/<project> -a search -q \"how is the hero styled\"";
+        return 1 ;
+      fi
+      rag_require_collection "${uri}" "${token}" || return 1 ;
+      rag_python --uri "${uri}" --token "${token}" search \
+        --collection "${RAG_COLLECTION}" --query "${QUERY}" --limit "${LIMIT}" ;
+      ;;
+    remove)
+      rag_require_collection "${uri}" "${token}" || return 1 ;
+      rag_python --uri "${uri}" --token "${token}" drop --collection "${RAG_COLLECTION}" ;
+      ;;
+    mcp-register|mcp-unregister|serve|status)
+      # The MCP server is per INSTANCE, not per collection: its tools list and search every
+      # collection in the database, so registering one index would be registering all of them.
+      PROJECT_NAME="${project}";
+      MCP_ACTION="${RAG_ACTION#mcp-}";
+      milvus_mcp ;
+      ;;
+    *)
+      echo "Unknown action '${RAG_ACTION}'." ; return 1 ;;
+  esac
+}

package/core/scripts/functions/fun-ragify.sh

@@ -0,0 +1,165 @@
+#!/usr/bin/env bash
+
+# Index a project in this workspace into the Milvus vector database — the RAG counterpart of
+# cmd-tools-graphify.sh.
+#
+# The two answer different questions, and the difference is the reason both exist:
+#
+#   graphify   parses code with tree-sitter into a graph. "What injects this service", "what
+#              extends this class". It reads no Markdown, no Twig, no *.module hooks and none of
+#              the Drupal wiring YAML — those are documents to it, and a code-only run drops them.
+#   ragify     indexes the TEXT of the same tree — including exactly those files — and retrieves
+#              passages. "Where is this explained", "which template renders that", "what does the
+#              recipe set for this field".
+#
+# Nothing here calls an LLM. Retrieval is Milvus's built-in BM25 function: the database tokenises
+# the text and builds the sparse vectors itself, so the default run sends nothing anywhere.
+# `--embedding openai` is the one mode that does, and it says so.
+
+# The chunks land in a collection named for the project, in whichever Milvus instance the rag
+# workspace holds. Deterministic (see rag_collection_name), so a re-index overwrites its own
+# collection and never anyone else's.
+
+# Third-party and generated trees, reusing exactly what graphify already excludes — a built site
+# build is 90% vendored code, and indexing it buries the project's own text under Symfony's.
+# The scope flag means the same thing here as there:
+#   custom   the project's own code only
+#   contrib  custom + contrib modules, themes and profiles   THE DEFAULT
+#   all      everything, core and vendor included
+function ragify_scope_excludes() {
+  graphify_scope_excludes "$1" ;
+}
+
+# Build (or rebuild) the index of a project in this workspace.
+function ragify_project() {
+  local project_path milvus_project uri token collection excludes embedding key model started;
+
+  project_path="${WORKSPACE_ROOT}/${doc_name}/${PROJECT_NAME}";
+  if [ ! -d "${project_path}" ] ; then
+    echo "No project ${PROJECT_NAME} in ${WORKSPACE_ROOT}/${doc_name}.";
+    return 1 ;
+  fi
+
+  # The database lives in the rag workspace, so its settings are read from there rather than from
+  # the calling workspace's own file — a module being indexed has no `milvus:` block of its own.
+  eval $(parse_yaml "${WORKSPACE_CONFIG}/workspace.rag.settings.yml" "rag_ws_");
+  milvus_project="$(milvus_setting "${rag_ws_milvus_project}")"; milvus_project="${milvus_project:-milvus}";
+  # rag_collection_name() reads the unprefixed name, so the naming rule is the same one
+  # cmd-tools-rag.sh applies when it reads the collection back.
+  rag_collection_prefix="${rag_ws_rag_collection_prefix}";
+
+  # Which RAG server. One shared instance is the default; --milvus binds this project to another
+  # one and remembers it, so every later index, search and MCP registration for it goes there.
+  if [ "${MILVUS_INSTANCE}" != '_settings_' ] ; then
+    if [ "${MILVUS_INSTANCE}" == 'default' ] ; then
+      rag_binding_set "${doc_name}" "${PROJECT_NAME}" "" ;
+      echo "Unbound ${doc_name}/${PROJECT_NAME} — it uses the default server again.";
+    elif printf '%s\n' "$(rag_instances)" | grep -qx "${MILVUS_INSTANCE}" ; then
+      rag_binding_set "${doc_name}" "${PROJECT_NAME}" "${MILVUS_INSTANCE}" ;
+      echo "Bound ${doc_name}/${PROJECT_NAME} to the ${MILVUS_INSTANCE} server.";
+    else
+      echo "No RAG server called '${MILVUS_INSTANCE}' in ${WORKSPACE_ROOT}/rag.";
+      echo "  Build it first: cd ${WORKSPACE_ROOT}/rag && bash cmd-milvus-project.sh ${MILVUS_INSTANCE}";
+      return 1 ;
+    fi
+  fi
+  milvus_project="$(rag_resolve_instance "${doc_name}" "${PROJECT_NAME}" '_settings_')";
+
+  # milvus_uri()/milvus_host_port() resolve against the rag workspace folder, which is not the
+  # folder this command was run from.
+  local doc_name_saved="${doc_name}";
+  doc_name="rag";
+  uri="$(milvus_uri "${milvus_project}")";
+  token="$(milvus_token)";
+  if ! milvus_require_running "${milvus_project}" ; then
+    doc_name="${doc_name_saved}";
+    return 1 ;
+  fi
+  doc_name="${doc_name_saved}";
+
+  collection="$(rag_collection_name "${doc_name}" "${PROJECT_NAME}")";
+  excludes="$(ragify_scope_excludes "${SCOPE}")";
+  [ "${ALL}" == 'yes' ] && excludes="";
+  # argparse.sh emits an nargs='+' option as a bash ARRAY, so plain ${EXCLUDE} is its FIRST
+  # element and `--exclude a b` quietly indexed b. [0] reads the sentinel on a scalar default too.
+  if [ "${EXCLUDE[0]}" != '_none_' ] ; then
+    excludes="${excludes} ${EXCLUDE[*]}";
+  fi
+
+  embedding="${EMBEDDING}";
+  if [ "${embedding}" == '_settings_' ] ; then
+    embedding="$(milvus_setting "${rag_ws_rag_embedding}")"; embedding="${embedding:-bm25}";
+  fi
+  model="$(milvus_setting "${rag_ws_rag_openai_model}")"; model="${model:-text-embedding-3-small}";
+
+  key="";
+  if [ "${embedding}" == 'openai' ] ; then
+    # The key the workspace already holds. A placeholder is not a key — sk-change-this would fail
+    # deep in the run with an authentication error rather than here with a sentence.
+    key="${OPENAI_API_KEY:-${ai_providers_openai_api_key}}";
+    case "${key}" in ''|CHANGE_ME|sk-change-this)
+      echo "--embedding openai needs an OpenAI key: set ai_providers.openai.api_key in";
+      echo "core/config/settings.yml, or export OPENAI_API_KEY for a one-off run.";
+      return 1 ;;
+    esac
+    echo "";
+    echo "Dense embeddings are ON: every indexed chunk of ${PROJECT_NAME} is sent to the OpenAI API";
+    echo "(${model}). The default BM25 mode sends nothing anywhere.";
+  fi
+
+  echo "";
+  echo "Indexing ${doc_name}/${PROJECT_NAME} into ${collection}";
+  echo "  Milvus     ${uri}";
+  echo "  Scope      ${SCOPE}$([ "${ALL}" == 'yes' ] && echo ' (--all: nothing excluded)')";
+  echo "  Retrieval  ${embedding}";
+  echo "";
+
+  started="$(date +%s)";
+  # shellcheck disable=SC2086
+  rag_python --uri "${uri}" --token "${token}" index \
+    --collection "${collection}" \
+    --path "${project_path}" \
+    --chunk-lines "$(milvus_setting "${rag_ws_rag_chunk_lines}")" \
+    --chunk-overlap "$(milvus_setting "${rag_ws_rag_chunk_overlap}")" \
+    --max-file-kb "$(milvus_setting "${rag_ws_rag_max_file_kb}")" \
+    --embedding "${embedding}" \
+    --openai-model "${model}" \
+    ${key:+--openai-key "${key}"} \
+    ${APPEND:+--append} \
+    --progress \
+    ${excludes:+--exclude ${excludes}} || return 1 ;
+
+  echo "";
+  echo "*---------------------------------------------------------------------*";
+  echo "  Indexed in $(( $(date +%s) - started ))s.";
+  echo "";
+  echo "  Ask it something:";
+  echo "    cd ${WORKSPACE_ROOT}/rag && bash cmd-tools-rag.sh ${doc_name}/${PROJECT_NAME} -a search -q \"...\"";
+  echo "  Give the whole database to the Claude Code CLI (all collections, one registration):";
+  echo "    cd ${WORKSPACE_ROOT}/rag && bash cmd-milvus-mcp.sh -a register";
+  echo "*---------------------------------------------------------------------*";
+}
+
+# Ask an existing index a question, from the project's own workspace folder — the same search
+# cmd-tools-rag.sh runs, without changing directory.
+function ragify_query_project() {
+  local milvus_project uri token collection doc_name_saved;
+
+  eval $(parse_yaml "${WORKSPACE_CONFIG}/workspace.rag.settings.yml" "rag_ws_");
+  rag_collection_prefix="${rag_ws_rag_collection_prefix}";
+  milvus_project="$(rag_resolve_instance "${doc_name}" "${PROJECT_NAME}" '_settings_')";
+
+  doc_name_saved="${doc_name}";
+  doc_name="rag";
+  uri="$(milvus_uri "${milvus_project}")";
+  token="$(milvus_token)";
+  if ! milvus_require_running "${milvus_project}" ; then
+    doc_name="${doc_name_saved}";
+    return 1 ;
+  fi
+  doc_name="${doc_name_saved}";
+
+  collection="$(rag_collection_name "${doc_name}" "${PROJECT_NAME}")";
+  rag_python --uri "${uri}" --token "${token}" search \
+    --collection "${collection}" --query "${QUERY}" --limit "${LIMIT}" ;
+}