@indigoai-us/hq-cloud 5.45.0 → 5.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +12 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +78 -12
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +27 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +17 -2
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +2 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +1 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +11 -1
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +1 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/object-io.d.ts +218 -0
- package/dist/object-io.d.ts.map +1 -0
- package/dist/object-io.js +588 -0
- package/dist/object-io.js.map +1 -0
- package/dist/object-io.test.d.ts +11 -0
- package/dist/object-io.test.d.ts.map +1 -0
- package/dist/object-io.test.js +568 -0
- package/dist/object-io.test.js.map +1 -0
- package/dist/s3.d.ts +37 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +207 -198
- package/dist/s3.js.map +1 -1
- package/dist/skill-telemetry.d.ts +107 -0
- package/dist/skill-telemetry.d.ts.map +1 -0
- package/dist/skill-telemetry.js +395 -0
- package/dist/skill-telemetry.js.map +1 -0
- package/dist/skill-telemetry.test.d.ts +2 -0
- package/dist/skill-telemetry.test.d.ts.map +1 -0
- package/dist/skill-telemetry.test.js +219 -0
- package/dist/skill-telemetry.test.js.map +1 -0
- package/dist/vault-client.d.ts +91 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +45 -0
- package/dist/vault-client.js.map +1 -1
- package/package.json +1 -1
- package/scripts/presign-transport-e2e.mjs +203 -0
- package/scripts/vault-rebaseline.sh +275 -0
- package/scripts/vault-rescue.sh +291 -0
- package/src/bin/sync-runner.test.ts +41 -0
- package/src/bin/sync-runner.ts +91 -13
- package/src/cli/share.test.ts +2 -0
- package/src/cli/share.ts +29 -2
- package/src/cli/sync-scope.test.ts +1 -0
- package/src/cli/sync.test.ts +1 -0
- package/src/cli/sync.ts +22 -1
- package/src/index.ts +16 -0
- package/src/object-io.test.ts +663 -0
- package/src/object-io.ts +782 -0
- package/src/s3.ts +259 -233
- package/src/skill-telemetry.test.ts +279 -0
- package/src/skill-telemetry.ts +499 -0
- package/src/vault-client.ts +135 -0
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# vault-rescue.sh — end-to-end vault rescue for a single HQ user.
|
|
3
|
+
#
|
|
4
|
+
# Pulls the user's S3 vault to a tmp dir, runs the hq-sync rescue script
|
|
5
|
+
# against it at a target hq-core version, and pushes the result back to
|
|
6
|
+
# S3 with the hq-symlink:<target> wire protocol intact. The whole flow
|
|
7
|
+
# replicates what the menubar app would do if the user clicked "Update HQ"
|
|
8
|
+
# locally — except the operator runs it remotely against any vault they
|
|
9
|
+
# have AWS S3 access to.
|
|
10
|
+
#
|
|
11
|
+
# Usage:
|
|
12
|
+
#
|
|
13
|
+
# vault-rescue.sh --prs <prs_*> --version <hq-version> [opts...]
|
|
14
|
+
#
|
|
15
|
+
# Required:
|
|
16
|
+
# --prs <prs_*> Entity UID (e.g. prs_01KQ7NTBRY8X2QAA4S8AAF26W6).
|
|
17
|
+
# Bucket name resolves to hq-vault-<lower-uid-w/-dashes>.
|
|
18
|
+
# --version <X.Y.Z> Target hq-core version. Becomes --ref v<X.Y.Z>.
|
|
19
|
+
#
|
|
20
|
+
# Optional:
|
|
21
|
+
# --source <owner/repo> Default: indigoai-us/hq-core (release repo, tagged).
|
|
22
|
+
# Override with indigoai-us/hq-core-staging for staging
|
|
23
|
+
# builds (no v-prefix; passes --ref main with the
|
|
24
|
+
# version used only for logging / future floor lookup).
|
|
25
|
+
# --dry-run Don't actually mutate S3. Pull still happens (so you
|
|
26
|
+
# can inspect the tmp dir), rescue runs --dry-run, push
|
|
27
|
+
# runs --dryrun. Combine with --keep-tmp to inspect
|
|
28
|
+
# what the rescue would have produced.
|
|
29
|
+
# --keep-tmp Don't delete the tmp dir on exit (useful for diffing).
|
|
30
|
+
# --rescue-script <path> Override path to hq-sync's replace-rescue.sh.
|
|
31
|
+
# Default resolves via $HQ_RESCUE_SCRIPT, then
|
|
32
|
+
# <this-script>/../../hq-sync/scripts/replace-rescue.sh.
|
|
33
|
+
# --profile <name> AWS profile. Default: $HQ_VAULT_AWS_PROFILE.
|
|
34
|
+
# --region <name> AWS region. Default: us-east-1.
|
|
35
|
+
# --tmp-dir <dir> Override the tmp dir path. Default: mktemp under /tmp.
|
|
36
|
+
# --floor-sha <40-hex> Forwarded to replace-rescue.sh; bypasses the
|
|
37
|
+
# vault's stamped last_sync_sha.
|
|
38
|
+
#
|
|
39
|
+
# Auth: uses the caller's AWS profile directly. The menubar app vends
|
|
40
|
+
# short-lived STS creds via vault-service; this script does not — it
|
|
41
|
+
# requires standing S3 perms on the target bucket. Operator/admin only.
|
|
42
|
+
#
|
|
43
|
+
# Output: live logs from each phase, plus a final summary. The pre-update
|
|
44
|
+
# safety snapshot from replace-rescue.sh lands at
|
|
45
|
+
# ~/.hq/backups/pre-update-<ts>/ on the OPERATOR's machine, not the user's.
|
|
46
|
+
|
|
47
|
+
set -euo pipefail
|
|
48
|
+
|
|
49
|
+
PRS=""
|
|
50
|
+
VERSION=""
|
|
51
|
+
SOURCE_REPO="indigoai-us/hq-core"
|
|
52
|
+
DRY_RUN=0
|
|
53
|
+
KEEP_TMP=0
|
|
54
|
+
RESCUE_SCRIPT="${HQ_RESCUE_SCRIPT:-}"
|
|
55
|
+
AWS_PROFILE_OVERRIDE="${HQ_VAULT_AWS_PROFILE:-}"
|
|
56
|
+
AWS_REGION_OVERRIDE="${HQ_VAULT_AWS_REGION:-us-east-1}"
|
|
57
|
+
TMP_DIR_OVERRIDE=""
|
|
58
|
+
FLOOR_SHA=""
|
|
59
|
+
|
|
60
|
+
# Default rescue script: discover relative to this file's location. Works
|
|
61
|
+
# when hq-cloud and hq-sync are sibling apps under hq-workspace; falls back
|
|
62
|
+
# to the env var when checked out elsewhere.
|
|
63
|
+
if [ -z "$RESCUE_SCRIPT" ]; then
|
|
64
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
65
|
+
CANDIDATE="$SCRIPT_DIR/../../hq-sync/scripts/replace-rescue.sh"
|
|
66
|
+
[ -f "$CANDIDATE" ] && RESCUE_SCRIPT="$CANDIDATE"
|
|
67
|
+
fi
|
|
68
|
+
|
|
69
|
+
usage() {
|
|
70
|
+
sed -n '2,45p' "$0"
|
|
71
|
+
exit 2
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
while [ $# -gt 0 ]; do
|
|
75
|
+
case "$1" in
|
|
76
|
+
--prs) PRS="$2"; shift 2 ;;
|
|
77
|
+
--version) VERSION="$2"; shift 2 ;;
|
|
78
|
+
--source) SOURCE_REPO="$2"; shift 2 ;;
|
|
79
|
+
--dry-run) DRY_RUN=1; shift ;;
|
|
80
|
+
--keep-tmp) KEEP_TMP=1; shift ;;
|
|
81
|
+
--rescue-script) RESCUE_SCRIPT="$2"; shift 2 ;;
|
|
82
|
+
--profile) AWS_PROFILE_OVERRIDE="$2"; shift 2 ;;
|
|
83
|
+
--region) AWS_REGION_OVERRIDE="$2"; shift 2 ;;
|
|
84
|
+
--tmp-dir) TMP_DIR_OVERRIDE="$2"; shift 2 ;;
|
|
85
|
+
--floor-sha) FLOOR_SHA="$2"; shift 2 ;;
|
|
86
|
+
-h|--help) usage ;;
|
|
87
|
+
*) echo "unknown arg: $1" >&2; usage ;;
|
|
88
|
+
esac
|
|
89
|
+
done
|
|
90
|
+
|
|
91
|
+
[ -n "$PRS" ] || { echo "error: --prs is required" >&2; usage; }
|
|
92
|
+
[ -n "$VERSION" ] || { echo "error: --version is required" >&2; usage; }
|
|
93
|
+
[ -n "$RESCUE_SCRIPT" ] || {
|
|
94
|
+
echo "error: replace-rescue.sh not found. Pass --rescue-script <path> or set HQ_RESCUE_SCRIPT." >&2
|
|
95
|
+
exit 2
|
|
96
|
+
}
|
|
97
|
+
[ -f "$RESCUE_SCRIPT" ] || {
|
|
98
|
+
echo "error: --rescue-script '$RESCUE_SCRIPT' is not a file" >&2
|
|
99
|
+
exit 2
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
case "$PRS" in
|
|
103
|
+
prs_*) ;;
|
|
104
|
+
*) echo "error: --prs must look like prs_*; got '$PRS'" >&2; exit 2 ;;
|
|
105
|
+
esac
|
|
106
|
+
|
|
107
|
+
# Bucket name convention (matches s3.ts entity → bucket mapping):
|
|
108
|
+
# prs_01ABC... → hq-vault-prs-01abc...
|
|
109
|
+
BUCKET="hq-vault-$(printf '%s' "$PRS" | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
|
|
110
|
+
|
|
111
|
+
# Construct rescue --ref. Tagged releases live in hq-core as v<version>;
|
|
112
|
+
# rolling staging uses main and treats --version as informational.
|
|
113
|
+
case "$SOURCE_REPO" in
|
|
114
|
+
*hq-core-staging) REF="main" ;;
|
|
115
|
+
*) REF="v$VERSION" ;;
|
|
116
|
+
esac
|
|
117
|
+
|
|
118
|
+
# AWS CLI prefix.
|
|
119
|
+
AWS_ARGS=()
|
|
120
|
+
[ -n "$AWS_PROFILE_OVERRIDE" ] && AWS_ARGS+=(--profile "$AWS_PROFILE_OVERRIDE")
|
|
121
|
+
AWS_ARGS+=(--region "$AWS_REGION_OVERRIDE")
|
|
122
|
+
|
|
123
|
+
# Tmp dir setup.
|
|
124
|
+
if [ -n "$TMP_DIR_OVERRIDE" ]; then
|
|
125
|
+
TMP_DIR="$TMP_DIR_OVERRIDE"
|
|
126
|
+
mkdir -p "$TMP_DIR"
|
|
127
|
+
else
|
|
128
|
+
TMP_DIR="$(mktemp -d -t hq-vault-rescue-XXXXXX)"
|
|
129
|
+
fi
|
|
130
|
+
|
|
131
|
+
# Trap MUST reference an unconditional global with default-empty fallback
|
|
132
|
+
# (set -u trips on undef refs in the trap body otherwise).
|
|
133
|
+
cleanup() {
|
|
134
|
+
local rc=$?
|
|
135
|
+
if [ "$KEEP_TMP" = "1" ]; then
|
|
136
|
+
echo "==> --keep-tmp set; preserving $TMP_DIR" >&2
|
|
137
|
+
elif [ -n "${TMP_DIR:-}" ] && [ -d "$TMP_DIR" ]; then
|
|
138
|
+
rm -rf "$TMP_DIR"
|
|
139
|
+
fi
|
|
140
|
+
return $rc
|
|
141
|
+
}
|
|
142
|
+
trap cleanup EXIT
|
|
143
|
+
|
|
144
|
+
cat <<EOF
|
|
145
|
+
==> vault-rescue
|
|
146
|
+
prs: $PRS
|
|
147
|
+
bucket: s3://$BUCKET
|
|
148
|
+
target: $SOURCE_REPO @ $REF (version $VERSION)
|
|
149
|
+
rescue script: $RESCUE_SCRIPT
|
|
150
|
+
tmp dir: $TMP_DIR
|
|
151
|
+
dry-run: $([ "$DRY_RUN" = "1" ] && echo "ON (no S3 mutation, no tmp mutation past pull)" || echo "off")
|
|
152
|
+
keep-tmp: $([ "$KEEP_TMP" = "1" ] && echo "yes" || echo "no")
|
|
153
|
+
EOF
|
|
154
|
+
|
|
155
|
+
# ---------- phase 1: pull ----------
|
|
156
|
+
echo "" >&2
|
|
157
|
+
echo "==> [1/3] pull s3://$BUCKET/ -> $TMP_DIR/" >&2
|
|
158
|
+
aws "${AWS_ARGS[@]}" s3 sync "s3://$BUCKET/" "$TMP_DIR/" --quiet
|
|
159
|
+
echo "==> pull done. files: $(find "$TMP_DIR" -type f | wc -l | tr -d ' ')" >&2
|
|
160
|
+
|
|
161
|
+
# ---------- phase 2: rescue ----------
|
|
162
|
+
echo "" >&2
|
|
163
|
+
echo "==> [2/3] rescue: $RESCUE_SCRIPT --source $SOURCE_REPO --ref $REF --cloud-update" >&2
|
|
164
|
+
|
|
165
|
+
RESCUE_ARGS=(
|
|
166
|
+
--hq-root "$TMP_DIR"
|
|
167
|
+
--source "$SOURCE_REPO"
|
|
168
|
+
--ref "$REF"
|
|
169
|
+
--cloud-update
|
|
170
|
+
--no-backup
|
|
171
|
+
--yes
|
|
172
|
+
)
|
|
173
|
+
# --no-backup is intentional: replace-rescue.sh's own pre-update snapshot
|
|
174
|
+
# would land at ~/.hq/backups/ on the OPERATOR's machine, not the vault
|
|
175
|
+
# owner's — so it doesn't help the user we're rescuing. Phase 1 already
|
|
176
|
+
# leaves the pulled tmp tree on disk (preserved with --keep-tmp), which IS
|
|
177
|
+
# the meaningful pre-rescue snapshot. Operators who want a hardened
|
|
178
|
+
# backup tarball it themselves before invoking this script.
|
|
179
|
+
# Requires hq-sync ≥0.6.4 (indigoai-us/hq-sync#170 — earlier versions
|
|
180
|
+
# return exit 1 on the --no-backup branch and kill us mid-flight).
|
|
181
|
+
[ "$DRY_RUN" = "1" ] && RESCUE_ARGS+=(--dry-run)
|
|
182
|
+
[ -n "$FLOOR_SHA" ] && RESCUE_ARGS+=(--floor-sha "$FLOOR_SHA")
|
|
183
|
+
|
|
184
|
+
bash "$RESCUE_SCRIPT" "${RESCUE_ARGS[@]}"
|
|
185
|
+
echo "==> rescue done." >&2
|
|
186
|
+
|
|
187
|
+
if [ "$DRY_RUN" = "1" ]; then
|
|
188
|
+
echo "" >&2
|
|
189
|
+
echo "==> [3/3] push: SKIPPED (--dry-run; rescue ran --dry-run too, no tmp mutation to push)" >&2
|
|
190
|
+
echo "==> vault-rescue complete (dry-run). Inspect $TMP_DIR." >&2
|
|
191
|
+
KEEP_TMP=1
|
|
192
|
+
exit 0
|
|
193
|
+
fi
|
|
194
|
+
|
|
195
|
+
# ---------- phase 3: push ----------
|
|
196
|
+
echo "" >&2
|
|
197
|
+
echo "==> [3/3] push $TMP_DIR/ -> s3://$BUCKET/" >&2
|
|
198
|
+
|
|
199
|
+
# Build a staging tree: regular files copied verbatim; local symlinks
|
|
200
|
+
# serialized as hq-symlink:<target> marker files (no trailing newline,
|
|
201
|
+
# per apps/hq-cloud/src/s3.ts:417). Skip a minimal denylist that should
|
|
202
|
+
# never round-trip through S3.
|
|
203
|
+
STAGING="$(mktemp -d -t hq-vault-rescue-stage-XXXXXX)"
|
|
204
|
+
add_staging_cleanup() {
|
|
205
|
+
rm -rf "${STAGING:-}"
|
|
206
|
+
}
|
|
207
|
+
# Chain cleanup: existing trap removes TMP_DIR; we also need STAGING gone.
|
|
208
|
+
trap '{ add_staging_cleanup; cleanup; }' EXIT
|
|
209
|
+
|
|
210
|
+
echo " staging at $STAGING" >&2
|
|
211
|
+
|
|
212
|
+
# Denylist (path prefixes that should never push back). This is INTENTIONALLY
|
|
213
|
+
# smaller than apps/hq-cloud/src/ignore.ts — the vault we just pulled IS the
|
|
214
|
+
# source of truth, so over-filtering would cause spurious deletes.
|
|
215
|
+
deny_path() {
|
|
216
|
+
case "$1" in
|
|
217
|
+
.git|.git/*) return 0 ;;
|
|
218
|
+
node_modules|node_modules/*) return 0 ;;
|
|
219
|
+
target|target/*) return 0 ;;
|
|
220
|
+
dist|dist/*) return 0 ;;
|
|
221
|
+
build|build/*) return 0 ;;
|
|
222
|
+
__pycache__|__pycache__/*) return 0 ;;
|
|
223
|
+
.venv|.venv/*|venv|venv/*) return 0 ;;
|
|
224
|
+
.DS_Store|*/.DS_Store) return 0 ;;
|
|
225
|
+
.env|.env.*) return 0 ;;
|
|
226
|
+
*.pyc|*.class) return 0 ;;
|
|
227
|
+
esac
|
|
228
|
+
return 1
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
n_files=0; n_symlinks=0; n_skipped=0
|
|
232
|
+
SYMLINK_LIST="$STAGING.symlinks.txt"
|
|
233
|
+
: > "$SYMLINK_LIST"
|
|
234
|
+
|
|
235
|
+
while IFS= read -r -d '' src; do
|
|
236
|
+
rel="${src#"$TMP_DIR/"}"
|
|
237
|
+
[ "$rel" = "$src" ] && continue
|
|
238
|
+
[ -z "$rel" ] && continue
|
|
239
|
+
if deny_path "$rel"; then
|
|
240
|
+
n_skipped=$((n_skipped + 1))
|
|
241
|
+
continue
|
|
242
|
+
fi
|
|
243
|
+
dest="$STAGING/$rel"
|
|
244
|
+
mkdir -p "$(dirname "$dest")"
|
|
245
|
+
if [ -L "$src" ]; then
|
|
246
|
+
target="$(readlink "$src")"
|
|
247
|
+
printf 'hq-symlink:%s' "$target" > "$dest"
|
|
248
|
+
printf '%s\n' "$rel" >> "$SYMLINK_LIST"
|
|
249
|
+
n_symlinks=$((n_symlinks + 1))
|
|
250
|
+
elif [ -f "$src" ]; then
|
|
251
|
+
cp -p "$src" "$dest"
|
|
252
|
+
n_files=$((n_files + 1))
|
|
253
|
+
fi
|
|
254
|
+
done < <(find "$TMP_DIR" \( -type d \( -name node_modules -o -name .git -o -name target -o -name dist -o -name build \) -prune \) -o \( \( -type f -o -type l \) -print0 \))
|
|
255
|
+
|
|
256
|
+
echo " staged files=$n_files symlinks=$n_symlinks skipped=$n_skipped" >&2
|
|
257
|
+
|
|
258
|
+
# Sync to S3 — `--size-only` because staging mtime=now (fresh write) and
|
|
259
|
+
# S3 LastModified reflects original upload, so the default mtime-based
|
|
260
|
+
# comparator would treat every staging file as newer and re-upload it.
|
|
261
|
+
echo " aws s3 sync $STAGING/ -> s3://$BUCKET/ --size-only --delete" >&2
|
|
262
|
+
aws "${AWS_ARGS[@]}" s3 sync "$STAGING/" "s3://$BUCKET/" --size-only --delete
|
|
263
|
+
|
|
264
|
+
# Stamp the hq-symlink metadata header on every symlink-marker object so
|
|
265
|
+
# the menubar pull materializes them as real symlinks (apps/hq-cloud/src/
|
|
266
|
+
# s3.ts:466-468 prefers header detection; body-sniff is fallback).
|
|
267
|
+
n_stamped=0
|
|
268
|
+
if [ -s "$SYMLINK_LIST" ]; then
|
|
269
|
+
echo " stamping hq-symlink-target metadata on $(wc -l < "$SYMLINK_LIST" | tr -d ' ') objects" >&2
|
|
270
|
+
while IFS= read -r rel; do
|
|
271
|
+
[ -z "$rel" ] && continue
|
|
272
|
+
aws "${AWS_ARGS[@]}" s3api put-object \
|
|
273
|
+
--bucket "$BUCKET" \
|
|
274
|
+
--key "$rel" \
|
|
275
|
+
--body "$STAGING/$rel" \
|
|
276
|
+
--metadata "hq-symlink-target=1" \
|
|
277
|
+
--content-type "text/plain" \
|
|
278
|
+
>/dev/null
|
|
279
|
+
n_stamped=$((n_stamped + 1))
|
|
280
|
+
done < "$SYMLINK_LIST"
|
|
281
|
+
fi
|
|
282
|
+
echo " stamped: $n_stamped" >&2
|
|
283
|
+
|
|
284
|
+
echo "" >&2
|
|
285
|
+
echo "==> vault-rescue complete." >&2
|
|
286
|
+
echo " bucket: s3://$BUCKET" >&2
|
|
287
|
+
echo " version: $VERSION ($SOURCE_REPO@$REF)" >&2
|
|
288
|
+
echo " files: $n_files" >&2
|
|
289
|
+
echo " symlinks: $n_symlinks" >&2
|
|
290
|
+
echo " stamped: $n_stamped" >&2
|
|
291
|
+
echo " skipped: $n_skipped" >&2
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
runRunnerWithLoop,
|
|
17
17
|
resolveDeletePolicy,
|
|
18
18
|
resolveSkipPersonal,
|
|
19
|
+
resolvePresignTransport,
|
|
19
20
|
routeChangeToTarget,
|
|
20
21
|
buildTargetedPushArgv,
|
|
21
22
|
resolvePullScope,
|
|
@@ -3126,6 +3127,46 @@ describe("resolveSkipPersonal", () => {
|
|
|
3126
3127
|
);
|
|
3127
3128
|
});
|
|
3128
3129
|
|
|
3130
|
+
// ---------------------------------------------------------------------------
|
|
3131
|
+
// resolvePresignTransport — @getindigo.ai rollout gate + env override
|
|
3132
|
+
// ---------------------------------------------------------------------------
|
|
3133
|
+
|
|
3134
|
+
describe("resolvePresignTransport", () => {
|
|
3135
|
+
it("ON for @getindigo.ai emails (case-insensitive), no override", () => {
|
|
3136
|
+
expect(resolvePresignTransport("me@getindigo.ai", undefined)).toBe(true);
|
|
3137
|
+
expect(resolvePresignTransport("ME@GetIndigo.AI", undefined)).toBe(true);
|
|
3138
|
+
});
|
|
3139
|
+
|
|
3140
|
+
it("OFF for non-getindigo emails, no override", () => {
|
|
3141
|
+
expect(resolvePresignTransport("me@example.com", undefined)).toBe(false);
|
|
3142
|
+
expect(resolvePresignTransport(undefined, undefined)).toBe(false);
|
|
3143
|
+
// A lookalike domain must not match the endsWith check loosely.
|
|
3144
|
+
expect(resolvePresignTransport("me@notgetindigo.ai.evil.com", undefined)).toBe(
|
|
3145
|
+
false,
|
|
3146
|
+
);
|
|
3147
|
+
});
|
|
3148
|
+
|
|
3149
|
+
it.each(["1", "true", "yes", "on", "ON", " True "])(
|
|
3150
|
+
"override '%s' forces ON even for a non-getindigo email",
|
|
3151
|
+
(val) => {
|
|
3152
|
+
expect(resolvePresignTransport("me@example.com", val)).toBe(true);
|
|
3153
|
+
},
|
|
3154
|
+
);
|
|
3155
|
+
|
|
3156
|
+
it.each(["0", "false", "no", "off", "OFF"])(
|
|
3157
|
+
"override '%s' forces OFF even for a getindigo email",
|
|
3158
|
+
(val) => {
|
|
3159
|
+
expect(resolvePresignTransport("me@getindigo.ai", val)).toBe(false);
|
|
3160
|
+
},
|
|
3161
|
+
);
|
|
3162
|
+
|
|
3163
|
+
it("blank/unrecognized override falls through to the email check", () => {
|
|
3164
|
+
expect(resolvePresignTransport("me@getindigo.ai", "")).toBe(true);
|
|
3165
|
+
expect(resolvePresignTransport("me@getindigo.ai", "maybe")).toBe(true);
|
|
3166
|
+
expect(resolvePresignTransport("me@example.com", "maybe")).toBe(false);
|
|
3167
|
+
});
|
|
3168
|
+
});
|
|
3169
|
+
|
|
3129
3170
|
// ---------------------------------------------------------------------------
|
|
3130
3171
|
// resolvePullScope (US-005) — effective download scope per company leg
|
|
3131
3172
|
// ---------------------------------------------------------------------------
|
package/src/bin/sync-runner.ts
CHANGED
|
@@ -91,7 +91,13 @@ import { share as defaultShare } from "../cli/share.js";
|
|
|
91
91
|
import type { ShareOptions, ShareResult } from "../cli/share.js";
|
|
92
92
|
import type { ConflictStrategy } from "../cli/conflict.js";
|
|
93
93
|
import type { UploadAuthor } from "../s3.js";
|
|
94
|
+
import {
|
|
95
|
+
setObjectIOFactory,
|
|
96
|
+
presignObjectIOFactory,
|
|
97
|
+
type PresignTransportClient,
|
|
98
|
+
} from "../object-io.js";
|
|
94
99
|
import { collectAndSendTelemetry } from "../telemetry.js";
|
|
100
|
+
import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
|
|
95
101
|
import { reindexAfterSync } from "../qmd-reindex.js";
|
|
96
102
|
import { describeError } from "../lib/describe-error.js";
|
|
97
103
|
import { getOrCreateMachineId } from "../lib/machine-id.js";
|
|
@@ -203,6 +209,27 @@ export function resolveSkipPersonal(flag: boolean): boolean {
|
|
|
203
209
|
return env === "1" || env === "true" || env === "yes";
|
|
204
210
|
}
|
|
205
211
|
|
|
212
|
+
/**
|
|
213
|
+
* Decide whether this session uses the presigned-URL transport.
|
|
214
|
+
*
|
|
215
|
+
* Rollout gate: ON for accounts whose verified email is `@getindigo.ai`.
|
|
216
|
+
* `HQ_SYNC_PRESIGN_TRANSPORT` overrides the email check in both directions
|
|
217
|
+
* (`1`/`true`/`yes`/`on` → force on, `0`/`false`/`no`/`off` → force off) so
|
|
218
|
+
* the transport can be exercised by non-getindigo testers or rolled back for
|
|
219
|
+
* getindigo accounts without a redeploy. An unset/blank override falls through
|
|
220
|
+
* to the email check; an unrecognized override value is ignored (email check
|
|
221
|
+
* wins) rather than silently forcing a state.
|
|
222
|
+
*/
|
|
223
|
+
export function resolvePresignTransport(
|
|
224
|
+
email: string | undefined,
|
|
225
|
+
override: string | undefined,
|
|
226
|
+
): boolean {
|
|
227
|
+
const o = (override ?? "").trim().toLowerCase();
|
|
228
|
+
if (o === "1" || o === "true" || o === "yes" || o === "on") return true;
|
|
229
|
+
if (o === "0" || o === "false" || o === "no" || o === "off") return false;
|
|
230
|
+
return typeof email === "string" && email.toLowerCase().endsWith("@getindigo.ai");
|
|
231
|
+
}
|
|
232
|
+
|
|
206
233
|
// Personal-vault scope (exclusion list + path computer) lives in
|
|
207
234
|
// `../personal-vault.ts` so the `hq sync` CLI and this runner share the same
|
|
208
235
|
// rules. Re-exported here for back-compat with any callers still importing
|
|
@@ -802,21 +829,31 @@ async function defaultCollectTelemetry(
|
|
|
802
829
|
hqRoot: string,
|
|
803
830
|
): Promise<void> {
|
|
804
831
|
if (clientIsStub) return;
|
|
832
|
+
|
|
833
|
+
// machineId: hq-cloud owns provisioning via `<hqRoot>/.hq/machine-id`
|
|
834
|
+
// (see `src/lib/machine-id.ts`). The resolver migrates forward from
|
|
835
|
+
// any legacy `~/.hq/menubar.json` value on first call, then becomes
|
|
836
|
+
// self-sufficient. On a clean Linux outpost (no menubar app), a fresh
|
|
837
|
+
// UUID is generated + persisted, so this row is attributable rather
|
|
838
|
+
// than collapsing onto the legacy `"unknown"` sentinel.
|
|
839
|
+
//
|
|
840
|
+
// installerVersion: callers (the Tauri menubar) set this when spawning
|
|
841
|
+
// the runner so the historical `installerVersion` dimension on CloudWatch
|
|
842
|
+
// keeps reporting the menubar version, not the runner's package version.
|
|
843
|
+
// CLI callers can leave it unset.
|
|
844
|
+
//
|
|
845
|
+
// Resolved once and shared by both telemetry passes below. If identity
|
|
846
|
+
// resolution itself throws, skip telemetry entirely.
|
|
847
|
+
let machineId: string;
|
|
848
|
+
let installerVersion: string;
|
|
805
849
|
try {
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
// than collapsing onto the legacy `"unknown"` sentinel.
|
|
812
|
-
const machineId = getOrCreateMachineId(hqRoot);
|
|
813
|
-
|
|
814
|
-
// installerVersion: callers (the Tauri menubar) set this when spawning
|
|
815
|
-
// the runner so the historical `installerVersion` dimension on
|
|
816
|
-
// CloudWatch keeps reporting the menubar version, not the runner's
|
|
817
|
-
// package version. CLI callers can leave it unset.
|
|
818
|
-
const installerVersion = process.env.HQ_INSTALLER_VERSION ?? "hq-cloud";
|
|
850
|
+
machineId = getOrCreateMachineId(hqRoot);
|
|
851
|
+
installerVersion = process.env.HQ_INSTALLER_VERSION ?? "hq-cloud";
|
|
852
|
+
} catch {
|
|
853
|
+
return;
|
|
854
|
+
}
|
|
819
855
|
|
|
856
|
+
try {
|
|
820
857
|
await collectAndSendTelemetry({
|
|
821
858
|
// The runtime guarantee here is that `clientIsStub === false` means
|
|
822
859
|
// `client` came from `new VaultClient(vaultConfig)` (see runRunner),
|
|
@@ -828,6 +865,21 @@ async function defaultCollectTelemetry(
|
|
|
828
865
|
} catch {
|
|
829
866
|
// Fire-and-forget; nothing escapes the boundary.
|
|
830
867
|
}
|
|
868
|
+
|
|
869
|
+
// Skill-invocation telemetry runs as an independent pass (own cursor, own
|
|
870
|
+
// endpoint) so a failure here can never affect token telemetry above.
|
|
871
|
+
try {
|
|
872
|
+
await collectAndSendSkillTelemetry({
|
|
873
|
+
client: client as unknown as Parameters<typeof collectAndSendSkillTelemetry>[0]["client"],
|
|
874
|
+
machineId,
|
|
875
|
+
installerVersion,
|
|
876
|
+
// Scope skill capture to the HQ project — only invocations whose cwd is
|
|
877
|
+
// the HQ root are emitted; skill usage in unrelated repos is excluded.
|
|
878
|
+
hqRoot,
|
|
879
|
+
});
|
|
880
|
+
} catch {
|
|
881
|
+
// Fire-and-forget; nothing escapes the boundary.
|
|
882
|
+
}
|
|
831
883
|
}
|
|
832
884
|
|
|
833
885
|
// ---------------------------------------------------------------------------
|
|
@@ -927,6 +979,32 @@ export async function runRunner(
|
|
|
927
979
|
? { userSub: claims.sub, email: claims.email }
|
|
928
980
|
: undefined;
|
|
929
981
|
|
|
982
|
+
// ---- transport selection (presigned-URL vs STS-direct-S3) -------------
|
|
983
|
+
// Default transport is unchanged: STS-vended credentials + direct S3 SDK.
|
|
984
|
+
// The presigned-URL transport (vault `list` + `presign` endpoints) is
|
|
985
|
+
// feature-flagged to @getindigo.ai accounts during rollout. The gate runs
|
|
986
|
+
// ONCE here — every s3.ts call in this session's fanout resolves through
|
|
987
|
+
// the installed factory. `HQ_SYNC_PRESIGN_TRANSPORT` is an explicit
|
|
988
|
+
// override (1/true → force on, 0/false → force off) that wins over the
|
|
989
|
+
// email check, so the flag can be flipped for testing or rollback without
|
|
990
|
+
// a redeploy. Setting the factory unconditionally (even to the default)
|
|
991
|
+
// keeps the choice deterministic if a prior run mutated module state.
|
|
992
|
+
const presignCapable = client as Partial<PresignTransportClient>;
|
|
993
|
+
if (
|
|
994
|
+
resolvePresignTransport(
|
|
995
|
+
claims?.email,
|
|
996
|
+
process.env.HQ_SYNC_PRESIGN_TRANSPORT,
|
|
997
|
+
) &&
|
|
998
|
+
typeof presignCapable.presign === "function" &&
|
|
999
|
+
typeof presignCapable.listFiles === "function"
|
|
1000
|
+
) {
|
|
1001
|
+
setObjectIOFactory(
|
|
1002
|
+
presignObjectIOFactory(presignCapable as PresignTransportClient),
|
|
1003
|
+
);
|
|
1004
|
+
} else {
|
|
1005
|
+
setObjectIOFactory(null);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
930
1008
|
// ---- resolve targets --------------------------------------------------
|
|
931
1009
|
let memberships: Pick<Membership, "companyUid">[];
|
|
932
1010
|
try {
|
package/src/cli/share.test.ts
CHANGED
|
@@ -21,6 +21,8 @@ vi.mock("../s3.js", () => ({
|
|
|
21
21
|
listRemoteFiles: vi.fn().mockResolvedValue([]),
|
|
22
22
|
deleteRemoteFile: vi.fn().mockResolvedValue(undefined),
|
|
23
23
|
headRemoteFile: vi.fn().mockResolvedValue(null),
|
|
24
|
+
primeObjectTransport: vi.fn().mockResolvedValue(undefined),
|
|
25
|
+
primeUploads: vi.fn().mockResolvedValue(undefined),
|
|
24
26
|
}));
|
|
25
27
|
|
|
26
28
|
// Mock readline so the interactive-conflict serialization test can observe
|
package/src/cli/share.ts
CHANGED
|
@@ -9,7 +9,15 @@ import * as fs from "fs";
|
|
|
9
9
|
import * as path from "path";
|
|
10
10
|
import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
|
|
11
11
|
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
uploadFile,
|
|
14
|
+
uploadSymlink,
|
|
15
|
+
headRemoteFile,
|
|
16
|
+
deleteRemoteFile,
|
|
17
|
+
downloadFile,
|
|
18
|
+
primeObjectTransport,
|
|
19
|
+
primeUploads,
|
|
20
|
+
} from "../s3.js";
|
|
13
21
|
import * as crypto from "crypto";
|
|
14
22
|
import type { UploadAuthor } from "../s3.js";
|
|
15
23
|
import {
|
|
@@ -784,6 +792,21 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
784
792
|
uploadItems.push(item);
|
|
785
793
|
}
|
|
786
794
|
|
|
795
|
+
// Batch pre-mint PUT URLs (+ the created-at HEADs) for the whole upload set,
|
|
796
|
+
// signing the SAME metadata the pool below computes so each task replays the
|
|
797
|
+
// cached headers and skips its own presign. Turns an N-file push from ~N
|
|
798
|
+
// presign calls into ceil(N/1000) GET + ceil(N/1000) PUT — keeping a bulk
|
|
799
|
+
// push under the 100/hr limit. No-op on the S3 SDK transport; best-effort.
|
|
800
|
+
await primeUploads(
|
|
801
|
+
ctx,
|
|
802
|
+
uploadItems.map((it) => ({
|
|
803
|
+
key: it.relativePath,
|
|
804
|
+
localPath: it.absolutePath,
|
|
805
|
+
isSymlink: it.kind === "symlink",
|
|
806
|
+
author: options.author,
|
|
807
|
+
})),
|
|
808
|
+
);
|
|
809
|
+
|
|
787
810
|
// Phase B: parallel upload pool. Each task runs the full per-item flow
|
|
788
811
|
// (HEAD + conflict + PUT + journal stamp + emit). Aborts flip the
|
|
789
812
|
// shared `aborted` flag and the pool stops draining the queue; tasks
|
|
@@ -1100,7 +1123,11 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1100
1123
|
// leg of `sync now` re-pulls naturally via the existing
|
|
1101
1124
|
// `hasRemoteChanged` path. Emit a dedicated event so UIs can
|
|
1102
1125
|
// surface the refusal without inferring it from absence.
|
|
1103
|
-
|
|
1126
|
+
// Batch pre-mint DELETE URLs so a large delete set is ~ceil(N/1000) presign
|
|
1127
|
+
// calls, not N. No-op on the S3 SDK transport; best-effort.
|
|
1128
|
+
const deleteKeys = [...deletePlan.toDelete, ...decommissionPlan];
|
|
1129
|
+
await primeObjectTransport(ctx, "delete", deleteKeys);
|
|
1130
|
+
for (const relativePath of deleteKeys) {
|
|
1104
1131
|
if (vaultConfig && isExpiringSoon(ctx.expiresAt)) {
|
|
1105
1132
|
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
1106
1133
|
}
|
package/src/cli/sync.test.ts
CHANGED
|
@@ -31,6 +31,7 @@ vi.mock("../s3.js", async () => {
|
|
|
31
31
|
listRemoteFiles: innerVi.fn().mockResolvedValue(remoteFiles),
|
|
32
32
|
deleteRemoteFile: innerVi.fn().mockResolvedValue(undefined),
|
|
33
33
|
headRemoteFile: innerVi.fn().mockResolvedValue(null),
|
|
34
|
+
primeObjectTransport: innerVi.fn().mockResolvedValue(undefined),
|
|
34
35
|
};
|
|
35
36
|
});
|
|
36
37
|
|
package/src/cli/sync.ts
CHANGED
|
@@ -10,7 +10,12 @@ import * as path from "path";
|
|
|
10
10
|
import type { VaultServiceConfig, SyncJournal } from "../types.js";
|
|
11
11
|
import type { SyncMode } from "../vault-client.js";
|
|
12
12
|
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
downloadFile,
|
|
15
|
+
listRemoteFiles,
|
|
16
|
+
headRemoteFile,
|
|
17
|
+
primeObjectTransport,
|
|
18
|
+
} from "../s3.js";
|
|
14
19
|
import type { RemoteFile } from "../s3.js";
|
|
15
20
|
import {
|
|
16
21
|
readJournal,
|
|
@@ -834,6 +839,18 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
834
839
|
// (inside the wrapper), so cross-file interleave is expected and the
|
|
835
840
|
// menubar's stream parser already handles it.
|
|
836
841
|
if (downloadItems.length > 0) {
|
|
842
|
+
// Batch pre-mint GET URLs for every download in one shot (chunked server-
|
|
843
|
+
// side) so the pool below — and the new-files HEAD enrichment that follows,
|
|
844
|
+
// which re-reads the same keys — reuse them instead of presigning per file.
|
|
845
|
+
// On a large initial pull this is the difference between ~ceil(N/100)
|
|
846
|
+
// presign calls and N (which would 429 past the 100-req/hr limit). No-op
|
|
847
|
+
// on the S3 SDK transport; best-effort (failure falls back to per-file).
|
|
848
|
+
await primeObjectTransport(
|
|
849
|
+
ctx,
|
|
850
|
+
"get",
|
|
851
|
+
downloadItems.map((d) => d.remoteFile.key),
|
|
852
|
+
);
|
|
853
|
+
|
|
837
854
|
const queue = [...downloadItems];
|
|
838
855
|
const inFlight: Set<Promise<unknown>> = new Set();
|
|
839
856
|
|
|
@@ -1022,6 +1039,10 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
1022
1039
|
// - HEAD throws transient → defensive skip + emit error.
|
|
1023
1040
|
// Bounded concurrency mirrors the new-files attribution pass above.
|
|
1024
1041
|
if (plan.tombstones.length > 0) {
|
|
1042
|
+
// Pre-mint GET URLs for the tombstone HEAD-verify probes below (headRemote
|
|
1043
|
+
// File presigns a GET), so a large delete set doesn't add N presign calls.
|
|
1044
|
+
await primeObjectTransport(ctx, "get", plan.tombstones);
|
|
1045
|
+
|
|
1025
1046
|
const HEAD_VERIFY_CONCURRENCY = 5;
|
|
1026
1047
|
const verified: string[] = [];
|
|
1027
1048
|
for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
|
package/src/index.ts
CHANGED
|
@@ -146,6 +146,8 @@ export type {
|
|
|
146
146
|
TelemetryOptInResponse,
|
|
147
147
|
UsageBatch,
|
|
148
148
|
UsageIngestResult,
|
|
149
|
+
SkillInvocationBatch,
|
|
150
|
+
SkillInvocationIngestResult,
|
|
149
151
|
// US-004 — browse-vs-sync membership sync-mode + ACL surface
|
|
150
152
|
SyncMode,
|
|
151
153
|
MembershipSyncConfig,
|
|
@@ -163,6 +165,20 @@ export type {
|
|
|
163
165
|
TelemetryClientSurface,
|
|
164
166
|
} from "./telemetry.js";
|
|
165
167
|
|
|
168
|
+
// Skill-invocation telemetry collector (`/v1/skill-invocations`). Reads the
|
|
169
|
+
// same Claude Code session logs as the token collector but extracts which
|
|
170
|
+
// skill / slash-command was invoked. Independent cursor; same opt-in gate.
|
|
171
|
+
export {
|
|
172
|
+
collectAndSendSkillTelemetry,
|
|
173
|
+
extractSkillEvents,
|
|
174
|
+
} from "./skill-telemetry.js";
|
|
175
|
+
export type {
|
|
176
|
+
CollectSkillTelemetryOptions,
|
|
177
|
+
CollectSkillTelemetryResult,
|
|
178
|
+
SkillEvent,
|
|
179
|
+
SkillTelemetryClientSurface,
|
|
180
|
+
} from "./skill-telemetry.js";
|
|
181
|
+
|
|
166
182
|
// STS child vending (VLT-8)
|
|
167
183
|
export type {
|
|
168
184
|
TaskAction,
|