@indigoai-us/hq-cloud 5.45.0 → 5.47.0

package/scripts/presign-transport-e2e.mjs

@@ -0,0 +1,203 @@
+/**
+ * Live E2E for the presigned-URL transport (hq-cloud feat/presign-transport).
+ *
+ * Drives the real s3.ts primitives through the PRESIGN factory against
+ * production (hqapi.getindigo.ai), exercising the exact code path that
+ * @getindigo.ai sync sessions will use: VaultClient.presign/listFiles +
+ * PresignObjectIO + fetch. Creates a scratch object under a clearly-marked
+ * prefix, round-trips it (put → list → head → get → symlink → delete), and
+ * deletes everything it created. Read-only on all real vault data.
+ *
+ * Run from the worktree root after `pnpm build`:
+ *   node scripts/presign-transport-e2e.mjs
+ */
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { VaultClient } from "../dist/vault-client.js";
+import {
+  setObjectIOFactory,
+  presignObjectIOFactory,
+} from "../dist/object-io.js";
+import {
+  uploadFile,
+  uploadSymlink,
+  downloadFile,
+  listRemoteFiles,
+  headRemoteFile,
+  deleteRemoteFile,
+} from "../dist/s3.js";
+
+const API = process.env.HQ_VAULT_API_URL ?? "https://hqapi.getindigo.ai";
+
+const results = [];
+function check(name, cond, detail = "") {
+  results.push({ name, ok: !!cond, detail });
+  console.log(`${cond ? "✓" : "✗"} ${name}${detail ? ` — ${detail}` : ""}`);
+}
+
+function loadAccessToken() {
+  const p = path.join(os.homedir(), ".hq", "cognito-tokens.json");
+  const t = JSON.parse(fs.readFileSync(p, "utf-8"));
+  const at = t.accessToken;
+  if (!at) throw new Error("no accessToken in cognito-tokens.json");
+  return at;
+}
+
+async function main() {
+  const token = loadAccessToken();
+  const vault = new VaultClient({ apiUrl: API, authToken: token });
+
+  // Resolve a company the caller can write to (owner/admin role).
+  const memberships = await vault.listMyMemberships();
+  const writable = memberships.find(
+    (m) => m.role === "owner" || m.role === "admin",
+  );
+  if (!writable) {
+    throw new Error(
+      `no owner/admin membership found (roles: ${memberships.map((m) => m.role).join(",") || "none"})`,
+    );
+  }
+  const companyUid = writable.companyUid;
+  let slug = companyUid;
+  try {
+    slug = (await vault.entity.get(companyUid)).slug ?? companyUid;
+  } catch {
+    /* display only */
+  }
+  console.log(`\nTarget: ${slug} (${companyUid}), role=${writable.role}\n`);
+
+  // Install the presign transport — the same selection runRunner makes for a
+  // @getindigo.ai session. ctx.credentials/bucketName/region are UNUSED on the
+  // presign path (only ctx.uid → companyUid matters), so dummy values are fine.
+  setObjectIOFactory(presignObjectIOFactory(vault));
+  const ctx = {
+    uid: companyUid,
+    slug,
+    bucketName: "(presign-unused)",
+    region: "us-east-1",
+    credentials: { accessKeyId: "x", secretAccessKey: "x", sessionToken: "x" },
+    expiresAt: "2099-01-01T00:00:00.000Z",
+  };
+
+  const stamp = Date.now().toString(36);
+  const prefix = `shared/_presign_e2e_${stamp}/`;
+  const fileKey = `${prefix}probe.txt`;
+  const linkKey = `${prefix}link`;
+  const author = { userSub: "e2e-probe-sub", email: "hassaan@getindigo.ai" };
+
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "presign-e2e-"));
+  const localUp = path.join(tmp, "up.txt");
+  const localDown = path.join(tmp, "down.txt");
+  const localLink = path.join(tmp, "link");
+  const content = `presign-transport probe ${stamp}\n${"x".repeat(2000)}`;
+  fs.writeFileSync(localUp, content);
+  fs.chmodSync(localUp, 0o640);
+
+  let allCreated = [];
+  try {
+    // ---- PUT (presigned, with metadata signed in) -----------------------
+    const up = await uploadFile(ctx, localUp, fileKey, author);
+    allCreated.push(fileKey);
+    check("uploadFile returns an etag", !!up.etag, up.etag);
+
+    // ---- LIST (presigned list, etag is load-bearing) --------------------
+    const listed = await listRemoteFiles(ctx, prefix);
+    const row = listed.find((o) => o.key === fileKey);
+    check("listRemoteFiles finds the uploaded key", !!row);
+    check(
+      "listed object carries a non-empty etag (#269)",
+      !!row && typeof row.etag === "string" && row.etag.length > 0,
+      row?.etag,
+    );
+    check(
+      "listed size matches uploaded bytes",
+      !!row && row.size === Buffer.byteLength(content),
+      `${row?.size} vs ${Buffer.byteLength(content)}`,
+    );
+
+    // ---- HEAD (presigned GET + header read) -----------------------------
+    const head = await headRemoteFile(ctx, fileKey);
+    check("headRemoteFile returns metadata", !!head && !!head.metadata);
+    check(
+      "metadata.created-by round-trips (signed PUT meta)",
+      head?.metadata?.["created-by"] === author.email,
+      head?.metadata?.["created-by"],
+    );
+    check(
+      "metadata.hq-mode round-trips as 640",
+      head?.metadata?.["hq-mode"] === "640",
+      head?.metadata?.["hq-mode"],
+    );
+    check(
+      "head etag matches list etag",
+      !!head && !!row && head.etag === row.etag,
+      `${head?.etag} vs ${row?.etag}`,
+    );
+
+    // ---- GET (presigned download, bytes + mode applied) -----------------
+    const dl = await downloadFile(ctx, fileKey, localDown);
+    const got = fs.readFileSync(localDown, "utf-8");
+    check("downloadFile bytes match upload", got === content);
+    check(
+      "downloaded file mode is 0640 (hq-mode applied)",
+      (fs.lstatSync(localDown).mode & 0o777) === 0o640,
+      "0" + (fs.lstatSync(localDown).mode & 0o777).toString(8),
+    );
+    check(
+      "downloadFile surfaces created-by metadata",
+      dl?.metadata?.["created-by"] === author.email,
+    );
+
+    // ---- SYMLINK round-trip --------------------------------------------
+    const ln = await uploadSymlink(ctx, "../target/path.txt", linkKey, author);
+    allCreated.push(linkKey);
+    check("uploadSymlink returns an etag", !!ln.etag);
+    await downloadFile(ctx, linkKey, localLink);
+    const st = fs.lstatSync(localLink);
+    check("downloaded symlink is a symlink", st.isSymbolicLink());
+    check(
+      "symlink target round-trips",
+      st.isSymbolicLink() && fs.readlinkSync(localLink) === "../target/path.txt",
+      st.isSymbolicLink() ? fs.readlinkSync(localLink) : "(not a link)",
+    );
+
+    // ---- DELETE (presigned) --------------------------------------------
+    await deleteRemoteFile(ctx, fileKey);
+    await deleteRemoteFile(ctx, linkKey);
+    allCreated = [];
+    const afterFile = await headRemoteFile(ctx, fileKey);
+    check("headRemoteFile is null after delete", afterFile === null);
+    const afterList = await listRemoteFiles(ctx, prefix);
+    check(
+      "listRemoteFiles empty after delete",
+      afterList.filter((o) => o.key.startsWith(prefix)).length === 0,
+      `${afterList.length} remaining`,
+    );
+  } finally {
+    // Best-effort cleanup of anything left if an assertion threw mid-run.
+    for (const k of allCreated) {
+      try {
+        await deleteRemoteFile(ctx, k);
+      } catch {
+        /* ignore */
+      }
+    }
+    fs.rmSync(tmp, { recursive: true, force: true });
+  }
+
+  const failed = results.filter((r) => !r.ok);
+  console.log(
+    `\n${results.length - failed.length}/${results.length} checks passed`,
+  );
+  if (failed.length > 0) {
+    console.log("FAILED:", failed.map((f) => f.name).join("; "));
+    process.exit(1);
+  }
+  console.log("ALL GREEN");
+}
+
+main().catch((err) => {
+  console.error("E2E error:", err);
+  process.exit(2);
+});

package/scripts/vault-rebaseline.sh

@@ -0,0 +1,275 @@
+#!/usr/bin/env bash
+# vault-rebaseline.sh — overlay hq-core@vX.Y.Z baseline content onto a vault.
+#
+# For vaults that pre-date core/core.yaml (pre-v15 partial cloud snapshots
+# missing the companies/ + personal/ structure). `replace-rescue.sh` refuses
+# to operate on those vaults because its safety gate requires HQ-root layout;
+# this script provides a narrower, additive alternative: push the v15
+# baseline tree (.claude/, .agents/, .codex/, core/, companies/_template/,
+# personal/, etc.) into the vault, preserving everything the user already
+# has there. Symlinks are serialized to `hq-symlink:<target>` markers so
+# they round-trip cleanly through hq-sync's protocol.
+#
+# Purely additive by default — no `--delete`. Their v14-era root files
+# (CHANGELOG.md, RELEASE-NOTES-v14.0.0.md, etc.) remain in S3 until they
+# either upgrade locally (menubar Update HQ) or we run a follow-up cleanup.
+# Pass `--delete-v14-cruft` to remove a hardcoded set of known v14-only
+# root files in the same run.
+#
+# Usage:
+#
+#   vault-rebaseline.sh --prs <prs_*> [--version X.Y.Z] [opts...]
+#
+# Required:
+#   --prs <prs_*>          Entity UID. Bucket = hq-vault-<lower-uid-w/-dashes>.
+#
+# Optional:
+#   --version <X.Y.Z>      Default 15.0.4 (current latest release).
+#   --source <owner/repo>  Default indigoai-us/hq-core. Override e.g. with
+#                          indigoai-us/hq-core-staging (would use --ref main
+#                          when version is not a tag).
+#   --dry-run              Show plan, don't push.
+#   --keep-tmp             Preserve the baseline clone + staging dirs.
+#   --delete-v14-cruft     Also delete a known set of v14 root files
+#                          (CHANGELOG.md, README.md, LICENSE, setup.sh,
+#                          RELEASE-NOTES-v14.0.0.md, MIGRATION-v11.md,
+#                          USER-GUIDE.md, CONTRIBUTING.md) so their
+#                          local install drops them on next sync.
+#   --profile/--region     AWS config. Default $HQ_VAULT_AWS_PROFILE /
+#                          us-east-1.
+#
+# Auth: uses the operator's standing AWS S3 perms directly. Operator/admin
+# only — vault-service STS vending is bypassed.
+
+set -euo pipefail
+
+PRS=""
+VERSION="15.0.4"
+SOURCE_REPO="indigoai-us/hq-core"
+DRY_RUN=0
+KEEP_TMP=0
+DELETE_CRUFT=0
+AWS_PROFILE_OVERRIDE="${HQ_VAULT_AWS_PROFILE:-}"
+AWS_REGION_OVERRIDE="${HQ_VAULT_AWS_REGION:-us-east-1}"
+
+usage() {
+  sed -n '2,40p' "$0"
+  exit 2
+}
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --prs)               PRS="$2"; shift 2 ;;
+    --version)           VERSION="$2"; shift 2 ;;
+    --source)            SOURCE_REPO="$2"; shift 2 ;;
+    --dry-run)           DRY_RUN=1; shift ;;
+    --keep-tmp)          KEEP_TMP=1; shift ;;
+    --delete-v14-cruft)  DELETE_CRUFT=1; shift ;;
+    --profile)           AWS_PROFILE_OVERRIDE="$2"; shift 2 ;;
+    --region)            AWS_REGION_OVERRIDE="$2"; shift 2 ;;
+    -h|--help)           usage ;;
+    *) echo "unknown arg: $1" >&2; usage ;;
+  esac
+done
+
+[ -n "$PRS" ] || { echo "error: --prs is required" >&2; usage; }
+case "$PRS" in
+  prs_*) ;;
+  *) echo "error: --prs must look like prs_*; got '$PRS'" >&2; exit 2 ;;
+esac
+
+BUCKET="hq-vault-$(printf '%s' "$PRS" | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
+
+# Construct git ref. Tagged releases live in hq-core as v<version>; staging
+# uses main and treats --version as informational.
+case "$SOURCE_REPO" in
+  *hq-core-staging) REF="main" ;;
+  *)                REF="v$VERSION" ;;
+esac
+
+AWS_ARGS=()
+[ -n "$AWS_PROFILE_OVERRIDE" ] && AWS_ARGS+=(--profile "$AWS_PROFILE_OVERRIDE")
+AWS_ARGS+=(--region "$AWS_REGION_OVERRIDE")
+
+# Tmp dirs.
+BASELINE="$(mktemp -d -t hq-rebaseline-baseline-XXXXXX)"
+STAGING="$(mktemp -d -t hq-rebaseline-stage-XXXXXX)"
+
+cleanup() {
+  local rc=$?
+  if [ "$KEEP_TMP" = "1" ]; then
+    echo "==> --keep-tmp set; preserving $BASELINE and $STAGING" >&2
+  else
+    rm -rf "${BASELINE:-}" "${STAGING:-}"
+  fi
+  return $rc
+}
+trap cleanup EXIT
+
+cat <<EOF
+==> vault-rebaseline
+    prs:         $PRS
+    bucket:      s3://$BUCKET
+    baseline:    $SOURCE_REPO @ $REF (version $VERSION)
+    baseline tmp: $BASELINE
+    staging tmp: $STAGING
+    dry-run:     $([ "$DRY_RUN" = "1" ] && echo "ON" || echo "off")
+    delete v14:  $([ "$DELETE_CRUFT" = "1" ] && echo "ON" || echo "off")
+    keep-tmp:    $([ "$KEEP_TMP" = "1" ] && echo "yes" || echo "no")
+EOF
+
+# ---------- phase 1: clone baseline ----------
+echo "" >&2
+echo "==> [1/3] clone $SOURCE_REPO@$REF (depth=1)" >&2
+git clone --quiet --depth 1 --branch "$REF" "https://github.com/$SOURCE_REPO.git" "$BASELINE" 2>&1 | tail -3 || {
+  echo "error: clone failed. Is $SOURCE_REPO@$REF a valid ref?" >&2
+  exit 1
+}
+echo "==> clone done. baseline files: $(find "$BASELINE" -type f | wc -l | tr -d ' ') (excl .git)" >&2
+
+# ---------- phase 2: stage with symlink serialization ----------
+echo "" >&2
+echo "==> [2/3] stage baseline tree (serialize symlinks → hq-symlink: markers)" >&2
+
+n_files=0; n_syms=0
+SYMLIST="$STAGING.symlinks.txt"
+: > "$SYMLIST"
+
+while IFS= read -r -d '' src; do
+  rel="${src#"$BASELINE/"}"
+  [ "$rel" = "$src" ] && continue
+  [ -z "$rel" ] && continue
+
+  # Skip the clone's .git directory (we're shipping just the tree, not the repo).
+  case "$rel" in
+    .git|.git/*) continue ;;
+  esac
+
+  dest="$STAGING/$rel"
+  mkdir -p "$(dirname "$dest")"
+
+  if [ -L "$src" ]; then
+    # Read the symlink's literal target string (apps/hq-cloud/src/s3.ts:417).
+    target="$(readlink "$src")"
+    printf 'hq-symlink:%s' "$target" > "$dest"
+    printf '%s\n' "$rel" >> "$SYMLIST"
+    n_syms=$((n_syms + 1))
+  elif [ -f "$src" ]; then
+    cp -p "$src" "$dest"
+    n_files=$((n_files + 1))
+  fi
+done < <(find "$BASELINE" \( -type d -name .git -prune \) -o \( \( -type f -o -type l \) -print0 \))
+
+echo "    staged: $n_files files, $n_syms symlinks" >&2
+
+# Strip files from staging that are KNOWN user-state — baseline ships
+# default values for these but the vault almost certainly has the user's
+# own customizations. Overwriting them would clobber config that lives
+# nowhere else (the menubar's hq-cloud/ignore.ts excludes some of these
+# from cloud sync, but earlier vault versions uploaded them).
+#
+# Each entry is a relative path inside the staging tree. Removed BEFORE
+# `aws s3 sync` runs so the file is not even a candidate for upload.
+PRESERVE_USER_STATE=(
+  # Company registry — user-owned, registers their connected companies'
+  # cloud_uid + bucket_name. v15 baseline ships an empty stub.
+  "companies/manifest.yaml"
+  # Claude Code prefs — user-customized model, env vars, hooks. v15 ships
+  # opinionated defaults that would overwrite e.g. MAX_THINKING_TOKENS,
+  # CLAUDE_CODE_SUBAGENT_MODEL, hook routing.
+  ".claude/settings.json"
+  # Obsidian UI state — pane layout, bookmarks, graph view. v15 ships
+  # defaults; users with active Obsidian have their own.
+  ".obsidian/workspace.json"
+  ".obsidian/graph.json"
+  ".obsidian/bookmarks.json"
+  ".obsidian/app.json"
+  ".obsidian/appearance.json"
+  ".obsidian/core-plugins.json"
+)
+n_preserved=0
+for rel in "${PRESERVE_USER_STATE[@]}"; do
+  if [ -f "$STAGING/$rel" ]; then
+    rm -f "$STAGING/$rel"
+    n_preserved=$((n_preserved + 1))
+    echo "    preserving user state (skipped from upload): $rel" >&2
+  fi
+done
+
+# ---------- phase 3: push ----------
+echo "" >&2
+echo "==> [3/3] aws s3 sync $STAGING/ → s3://$BUCKET/" >&2
+
+# `--size-only` because staging mtime is fresh and S3 LastModified reflects
+# original upload time. Default mtime-based comparison would re-upload every
+# content-stable file (same problem as vault-rescue.sh's push phase).
+#
+# No `--delete` by default. v14 baseline files coexist with v15 in S3 until
+# either the user upgrades locally (menubar Update HQ) or --delete-v14-cruft
+# is passed (handled below as a separate rm pass for a narrow allowlist).
+SYNC_ARGS=("s3" "sync" "$STAGING/" "s3://$BUCKET/" "--size-only")
+[ "$DRY_RUN" = "1" ] && SYNC_ARGS+=("--dryrun")
+
+aws "${AWS_ARGS[@]}" "${SYNC_ARGS[@]}"
+
+# Stamp the hq-symlink-target metadata header on every marker so the menubar
+# pull materializes them as real symlinks (apps/hq-cloud/src/s3.ts:466-468
+# prefers header detection; body sniff is fallback).
+n_stamped=0
+if [ "$DRY_RUN" = "0" ] && [ -s "$SYMLIST" ]; then
+  echo "" >&2
+  echo "    stamping hq-symlink-target metadata on $(wc -l < "$SYMLIST" | tr -d ' ') markers" >&2
+  while IFS= read -r rel; do
+    [ -z "$rel" ] && continue
+    aws "${AWS_ARGS[@]}" s3api put-object \
+      --bucket "$BUCKET" \
+      --key "$rel" \
+      --body "$STAGING/$rel" \
+      --metadata "hq-symlink-target=1" \
+      --content-type "text/plain" \
+      >/dev/null
+    n_stamped=$((n_stamped + 1))
+  done < "$SYMLIST"
+  echo "    stamped: $n_stamped" >&2
+fi
+
+# Optional: delete v14-era root files that don't exist in v15. Narrow
+# allowlist — files we can confidently say were always release-shipped and
+# would never be user-customized. Everything else stays.
+n_deleted=0
+if [ "$DELETE_CRUFT" = "1" ]; then
+  echo "" >&2
+  echo "    deleting v14 root-file cruft" >&2
+  V14_CRUFT=(
+    "CHANGELOG.md"
+    "README.md"
+    "LICENSE"
+    "setup.sh"
+    "RELEASE-NOTES-v14.0.0.md"
+    "MIGRATION-v11.md"
+    "USER-GUIDE.md"
+    "CONTRIBUTING.md"
+  )
+  for key in "${V14_CRUFT[@]}"; do
+    # Probe first — only delete if it actually exists.
+    if aws "${AWS_ARGS[@]}" s3api head-object --bucket "$BUCKET" --key "$key" >/dev/null 2>&1; then
+      if [ "$DRY_RUN" = "1" ]; then
+        echo "    would delete: $key" >&2
+      else
+        aws "${AWS_ARGS[@]}" s3api delete-object --bucket "$BUCKET" --key "$key" >/dev/null
+        echo "    deleted: $key" >&2
+      fi
+      n_deleted=$((n_deleted + 1))
+    fi
+  done
+fi
+
+echo "" >&2
+echo "==> vault-rebaseline complete." >&2
+echo "    bucket:        s3://$BUCKET" >&2
+echo "    version:       $VERSION ($SOURCE_REPO@$REF)" >&2
+echo "    pushed:        $n_files files, $n_syms symlinks (gross — preserved excluded)" >&2
+echo "    user-state preserved: $n_preserved files" >&2
+echo "    stamped:       $n_stamped" >&2
+echo "    deleted:       $n_deleted (v14 cruft)" >&2
+exit 0