@indigoai-us/hq-cloud 5.46.0 → 5.47.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner.d.ts +12 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +39 -0
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +27 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +17 -2
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +2 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync-scope.test.js +1 -0
- package/dist/cli/sync-scope.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +11 -1
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +1 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/object-io.d.ts +218 -0
- package/dist/object-io.d.ts.map +1 -0
- package/dist/object-io.js +588 -0
- package/dist/object-io.js.map +1 -0
- package/dist/object-io.test.d.ts +11 -0
- package/dist/object-io.test.d.ts.map +1 -0
- package/dist/object-io.test.js +568 -0
- package/dist/object-io.test.js.map +1 -0
- package/dist/s3.d.ts +37 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +225 -201
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +21 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/vault-client.d.ts +68 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +35 -0
- package/dist/vault-client.js.map +1 -1
- package/package.json +1 -1
- package/scripts/presign-transport-e2e.mjs +203 -0
- package/scripts/vault-rebaseline.sh +275 -0
- package/scripts/vault-rescue.sh +8 -0
- package/src/bin/sync-runner.test.ts +41 -0
- package/src/bin/sync-runner.ts +52 -0
- package/src/cli/share.test.ts +2 -0
- package/src/cli/share.ts +29 -2
- package/src/cli/sync-scope.test.ts +1 -0
- package/src/cli/sync.test.ts +1 -0
- package/src/cli/sync.ts +22 -1
- package/src/object-io.test.ts +663 -0
- package/src/object-io.ts +782 -0
- package/src/s3.test.ts +24 -0
- package/src/s3.ts +277 -237
- package/src/vault-client.ts +101 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live E2E for the presigned-URL transport (hq-cloud feat/presign-transport).
|
|
3
|
+
*
|
|
4
|
+
* Drives the real s3.ts primitives through the PRESIGN factory against
|
|
5
|
+
* production (hqapi.getindigo.ai), exercising the exact code path that
|
|
6
|
+
* @getindigo.ai sync sessions will use: VaultClient.presign/listFiles +
|
|
7
|
+
* PresignObjectIO + fetch. Creates a scratch object under a clearly-marked
|
|
8
|
+
* prefix, round-trips it (put → list → head → get → symlink → delete), and
|
|
9
|
+
* deletes everything it created. Read-only on all real vault data.
|
|
10
|
+
*
|
|
11
|
+
* Run from the worktree root after `pnpm build`:
|
|
12
|
+
* node scripts/presign-transport-e2e.mjs
|
|
13
|
+
*/
|
|
14
|
+
import * as fs from "fs";
|
|
15
|
+
import * as os from "os";
|
|
16
|
+
import * as path from "path";
|
|
17
|
+
import { VaultClient } from "../dist/vault-client.js";
|
|
18
|
+
import {
|
|
19
|
+
setObjectIOFactory,
|
|
20
|
+
presignObjectIOFactory,
|
|
21
|
+
} from "../dist/object-io.js";
|
|
22
|
+
import {
|
|
23
|
+
uploadFile,
|
|
24
|
+
uploadSymlink,
|
|
25
|
+
downloadFile,
|
|
26
|
+
listRemoteFiles,
|
|
27
|
+
headRemoteFile,
|
|
28
|
+
deleteRemoteFile,
|
|
29
|
+
} from "../dist/s3.js";
|
|
30
|
+
|
|
31
|
+
const API = process.env.HQ_VAULT_API_URL ?? "https://hqapi.getindigo.ai";
|
|
32
|
+
|
|
33
|
+
const results = [];
|
|
34
|
+
function check(name, cond, detail = "") {
|
|
35
|
+
results.push({ name, ok: !!cond, detail });
|
|
36
|
+
console.log(`${cond ? "✓" : "✗"} ${name}${detail ? ` — ${detail}` : ""}`);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function loadAccessToken() {
|
|
40
|
+
const p = path.join(os.homedir(), ".hq", "cognito-tokens.json");
|
|
41
|
+
const t = JSON.parse(fs.readFileSync(p, "utf-8"));
|
|
42
|
+
const at = t.accessToken;
|
|
43
|
+
if (!at) throw new Error("no accessToken in cognito-tokens.json");
|
|
44
|
+
return at;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function main() {
|
|
48
|
+
const token = loadAccessToken();
|
|
49
|
+
const vault = new VaultClient({ apiUrl: API, authToken: token });
|
|
50
|
+
|
|
51
|
+
// Resolve a company the caller can write to (owner/admin role).
|
|
52
|
+
const memberships = await vault.listMyMemberships();
|
|
53
|
+
const writable = memberships.find(
|
|
54
|
+
(m) => m.role === "owner" || m.role === "admin",
|
|
55
|
+
);
|
|
56
|
+
if (!writable) {
|
|
57
|
+
throw new Error(
|
|
58
|
+
`no owner/admin membership found (roles: ${memberships.map((m) => m.role).join(",") || "none"})`,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
const companyUid = writable.companyUid;
|
|
62
|
+
let slug = companyUid;
|
|
63
|
+
try {
|
|
64
|
+
slug = (await vault.entity.get(companyUid)).slug ?? companyUid;
|
|
65
|
+
} catch {
|
|
66
|
+
/* display only */
|
|
67
|
+
}
|
|
68
|
+
console.log(`\nTarget: ${slug} (${companyUid}), role=${writable.role}\n`);
|
|
69
|
+
|
|
70
|
+
// Install the presign transport — the same selection runRunner makes for a
|
|
71
|
+
// @getindigo.ai session. ctx.credentials/bucketName/region are UNUSED on the
|
|
72
|
+
// presign path (only ctx.uid → companyUid matters), so dummy values are fine.
|
|
73
|
+
setObjectIOFactory(presignObjectIOFactory(vault));
|
|
74
|
+
const ctx = {
|
|
75
|
+
uid: companyUid,
|
|
76
|
+
slug,
|
|
77
|
+
bucketName: "(presign-unused)",
|
|
78
|
+
region: "us-east-1",
|
|
79
|
+
credentials: { accessKeyId: "x", secretAccessKey: "x", sessionToken: "x" },
|
|
80
|
+
expiresAt: "2099-01-01T00:00:00.000Z",
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
const stamp = Date.now().toString(36);
|
|
84
|
+
const prefix = `shared/_presign_e2e_${stamp}/`;
|
|
85
|
+
const fileKey = `${prefix}probe.txt`;
|
|
86
|
+
const linkKey = `${prefix}link`;
|
|
87
|
+
const author = { userSub: "e2e-probe-sub", email: "hassaan@getindigo.ai" };
|
|
88
|
+
|
|
89
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "presign-e2e-"));
|
|
90
|
+
const localUp = path.join(tmp, "up.txt");
|
|
91
|
+
const localDown = path.join(tmp, "down.txt");
|
|
92
|
+
const localLink = path.join(tmp, "link");
|
|
93
|
+
const content = `presign-transport probe ${stamp}\n${"x".repeat(2000)}`;
|
|
94
|
+
fs.writeFileSync(localUp, content);
|
|
95
|
+
fs.chmodSync(localUp, 0o640);
|
|
96
|
+
|
|
97
|
+
let allCreated = [];
|
|
98
|
+
try {
|
|
99
|
+
// ---- PUT (presigned, with metadata signed in) -----------------------
|
|
100
|
+
const up = await uploadFile(ctx, localUp, fileKey, author);
|
|
101
|
+
allCreated.push(fileKey);
|
|
102
|
+
check("uploadFile returns an etag", !!up.etag, up.etag);
|
|
103
|
+
|
|
104
|
+
// ---- LIST (presigned list, etag is load-bearing) --------------------
|
|
105
|
+
const listed = await listRemoteFiles(ctx, prefix);
|
|
106
|
+
const row = listed.find((o) => o.key === fileKey);
|
|
107
|
+
check("listRemoteFiles finds the uploaded key", !!row);
|
|
108
|
+
check(
|
|
109
|
+
"listed object carries a non-empty etag (#269)",
|
|
110
|
+
!!row && typeof row.etag === "string" && row.etag.length > 0,
|
|
111
|
+
row?.etag,
|
|
112
|
+
);
|
|
113
|
+
check(
|
|
114
|
+
"listed size matches uploaded bytes",
|
|
115
|
+
!!row && row.size === Buffer.byteLength(content),
|
|
116
|
+
`${row?.size} vs ${Buffer.byteLength(content)}`,
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
// ---- HEAD (presigned GET + header read) -----------------------------
|
|
120
|
+
const head = await headRemoteFile(ctx, fileKey);
|
|
121
|
+
check("headRemoteFile returns metadata", !!head && !!head.metadata);
|
|
122
|
+
check(
|
|
123
|
+
"metadata.created-by round-trips (signed PUT meta)",
|
|
124
|
+
head?.metadata?.["created-by"] === author.email,
|
|
125
|
+
head?.metadata?.["created-by"],
|
|
126
|
+
);
|
|
127
|
+
check(
|
|
128
|
+
"metadata.hq-mode round-trips as 640",
|
|
129
|
+
head?.metadata?.["hq-mode"] === "640",
|
|
130
|
+
head?.metadata?.["hq-mode"],
|
|
131
|
+
);
|
|
132
|
+
check(
|
|
133
|
+
"head etag matches list etag",
|
|
134
|
+
!!head && !!row && head.etag === row.etag,
|
|
135
|
+
`${head?.etag} vs ${row?.etag}`,
|
|
136
|
+
);
|
|
137
|
+
|
|
138
|
+
// ---- GET (presigned download, bytes + mode applied) -----------------
|
|
139
|
+
const dl = await downloadFile(ctx, fileKey, localDown);
|
|
140
|
+
const got = fs.readFileSync(localDown, "utf-8");
|
|
141
|
+
check("downloadFile bytes match upload", got === content);
|
|
142
|
+
check(
|
|
143
|
+
"downloaded file mode is 0640 (hq-mode applied)",
|
|
144
|
+
(fs.lstatSync(localDown).mode & 0o777) === 0o640,
|
|
145
|
+
"0" + (fs.lstatSync(localDown).mode & 0o777).toString(8),
|
|
146
|
+
);
|
|
147
|
+
check(
|
|
148
|
+
"downloadFile surfaces created-by metadata",
|
|
149
|
+
dl?.metadata?.["created-by"] === author.email,
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
// ---- SYMLINK round-trip --------------------------------------------
|
|
153
|
+
const ln = await uploadSymlink(ctx, "../target/path.txt", linkKey, author);
|
|
154
|
+
allCreated.push(linkKey);
|
|
155
|
+
check("uploadSymlink returns an etag", !!ln.etag);
|
|
156
|
+
await downloadFile(ctx, linkKey, localLink);
|
|
157
|
+
const st = fs.lstatSync(localLink);
|
|
158
|
+
check("downloaded symlink is a symlink", st.isSymbolicLink());
|
|
159
|
+
check(
|
|
160
|
+
"symlink target round-trips",
|
|
161
|
+
st.isSymbolicLink() && fs.readlinkSync(localLink) === "../target/path.txt",
|
|
162
|
+
st.isSymbolicLink() ? fs.readlinkSync(localLink) : "(not a link)",
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
// ---- DELETE (presigned) --------------------------------------------
|
|
166
|
+
await deleteRemoteFile(ctx, fileKey);
|
|
167
|
+
await deleteRemoteFile(ctx, linkKey);
|
|
168
|
+
allCreated = [];
|
|
169
|
+
const afterFile = await headRemoteFile(ctx, fileKey);
|
|
170
|
+
check("headRemoteFile is null after delete", afterFile === null);
|
|
171
|
+
const afterList = await listRemoteFiles(ctx, prefix);
|
|
172
|
+
check(
|
|
173
|
+
"listRemoteFiles empty after delete",
|
|
174
|
+
afterList.filter((o) => o.key.startsWith(prefix)).length === 0,
|
|
175
|
+
`${afterList.length} remaining`,
|
|
176
|
+
);
|
|
177
|
+
} finally {
|
|
178
|
+
// Best-effort cleanup of anything left if an assertion threw mid-run.
|
|
179
|
+
for (const k of allCreated) {
|
|
180
|
+
try {
|
|
181
|
+
await deleteRemoteFile(ctx, k);
|
|
182
|
+
} catch {
|
|
183
|
+
/* ignore */
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const failed = results.filter((r) => !r.ok);
|
|
190
|
+
console.log(
|
|
191
|
+
`\n${results.length - failed.length}/${results.length} checks passed`,
|
|
192
|
+
);
|
|
193
|
+
if (failed.length > 0) {
|
|
194
|
+
console.log("FAILED:", failed.map((f) => f.name).join("; "));
|
|
195
|
+
process.exit(1);
|
|
196
|
+
}
|
|
197
|
+
console.log("ALL GREEN");
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
main().catch((err) => {
|
|
201
|
+
console.error("E2E error:", err);
|
|
202
|
+
process.exit(2);
|
|
203
|
+
});
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# vault-rebaseline.sh — overlay hq-core@vX.Y.Z baseline content onto a vault.
|
|
3
|
+
#
|
|
4
|
+
# For vaults that pre-date core/core.yaml (pre-v15 partial cloud snapshots
|
|
5
|
+
# missing the companies/ + personal/ structure). `replace-rescue.sh` refuses
|
|
6
|
+
# to operate on those vaults because its safety gate requires HQ-root layout;
|
|
7
|
+
# this script provides a narrower, additive alternative: push the v15
|
|
8
|
+
# baseline tree (.claude/, .agents/, .codex/, core/, companies/_template/,
|
|
9
|
+
# personal/, etc.) into the vault, preserving everything the user already
|
|
10
|
+
# has there. Symlinks are serialized to `hq-symlink:<target>` markers so
|
|
11
|
+
# they round-trip cleanly through hq-sync's protocol.
|
|
12
|
+
#
|
|
13
|
+
# Purely additive by default — no `--delete`. Their v14-era root files
|
|
14
|
+
# (CHANGELOG.md, RELEASE-NOTES-v14.0.0.md, etc.) remain in S3 until they
|
|
15
|
+
# either upgrade locally (menubar Update HQ) or we run a follow-up cleanup.
|
|
16
|
+
# Pass `--delete-v14-cruft` to remove a hardcoded set of known v14-only
|
|
17
|
+
# root files in the same run.
|
|
18
|
+
#
|
|
19
|
+
# Usage:
|
|
20
|
+
#
|
|
21
|
+
# vault-rebaseline.sh --prs <prs_*> [--version X.Y.Z] [opts...]
|
|
22
|
+
#
|
|
23
|
+
# Required:
|
|
24
|
+
# --prs <prs_*> Entity UID. Bucket = hq-vault-<lower-uid-w/-dashes>.
|
|
25
|
+
#
|
|
26
|
+
# Optional:
|
|
27
|
+
# --version <X.Y.Z> Default 15.0.4 (current latest release).
|
|
28
|
+
# --source <owner/repo> Default indigoai-us/hq-core. Override e.g. with
|
|
29
|
+
# indigoai-us/hq-core-staging (would use --ref main
|
|
30
|
+
# when version is not a tag).
|
|
31
|
+
# --dry-run Show plan, don't push.
|
|
32
|
+
# --keep-tmp Preserve the baseline clone + staging dirs.
|
|
33
|
+
# --delete-v14-cruft Also delete a known set of v14 root files
|
|
34
|
+
# (CHANGELOG.md, README.md, LICENSE, setup.sh,
|
|
35
|
+
# RELEASE-NOTES-v14.0.0.md, MIGRATION-v11.md,
|
|
36
|
+
# USER-GUIDE.md, CONTRIBUTING.md) so their
|
|
37
|
+
# local install drops them on next sync.
|
|
38
|
+
# --profile/--region AWS config. Default $HQ_VAULT_AWS_PROFILE /
|
|
39
|
+
# us-east-1.
|
|
40
|
+
#
|
|
41
|
+
# Auth: uses the operator's standing AWS S3 perms directly. Operator/admin
|
|
42
|
+
# only — vault-service STS vending is bypassed.
|
|
43
|
+
|
|
44
|
+
set -euo pipefail
|
|
45
|
+
|
|
46
|
+
PRS=""
|
|
47
|
+
VERSION="15.0.4"
|
|
48
|
+
SOURCE_REPO="indigoai-us/hq-core"
|
|
49
|
+
DRY_RUN=0
|
|
50
|
+
KEEP_TMP=0
|
|
51
|
+
DELETE_CRUFT=0
|
|
52
|
+
AWS_PROFILE_OVERRIDE="${HQ_VAULT_AWS_PROFILE:-}"
|
|
53
|
+
AWS_REGION_OVERRIDE="${HQ_VAULT_AWS_REGION:-us-east-1}"
|
|
54
|
+
|
|
55
|
+
usage() {
|
|
56
|
+
sed -n '2,40p' "$0"
|
|
57
|
+
exit 2
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
while [ $# -gt 0 ]; do
|
|
61
|
+
case "$1" in
|
|
62
|
+
--prs) PRS="$2"; shift 2 ;;
|
|
63
|
+
--version) VERSION="$2"; shift 2 ;;
|
|
64
|
+
--source) SOURCE_REPO="$2"; shift 2 ;;
|
|
65
|
+
--dry-run) DRY_RUN=1; shift ;;
|
|
66
|
+
--keep-tmp) KEEP_TMP=1; shift ;;
|
|
67
|
+
--delete-v14-cruft) DELETE_CRUFT=1; shift ;;
|
|
68
|
+
--profile) AWS_PROFILE_OVERRIDE="$2"; shift 2 ;;
|
|
69
|
+
--region) AWS_REGION_OVERRIDE="$2"; shift 2 ;;
|
|
70
|
+
-h|--help) usage ;;
|
|
71
|
+
*) echo "unknown arg: $1" >&2; usage ;;
|
|
72
|
+
esac
|
|
73
|
+
done
|
|
74
|
+
|
|
75
|
+
[ -n "$PRS" ] || { echo "error: --prs is required" >&2; usage; }
|
|
76
|
+
case "$PRS" in
|
|
77
|
+
prs_*) ;;
|
|
78
|
+
*) echo "error: --prs must look like prs_*; got '$PRS'" >&2; exit 2 ;;
|
|
79
|
+
esac
|
|
80
|
+
|
|
81
|
+
BUCKET="hq-vault-$(printf '%s' "$PRS" | tr '[:upper:]' '[:lower:]' | tr '_' '-')"
|
|
82
|
+
|
|
83
|
+
# Construct git ref. Tagged releases live in hq-core as v<version>; staging
|
|
84
|
+
# uses main and treats --version as informational.
|
|
85
|
+
case "$SOURCE_REPO" in
|
|
86
|
+
*hq-core-staging) REF="main" ;;
|
|
87
|
+
*) REF="v$VERSION" ;;
|
|
88
|
+
esac
|
|
89
|
+
|
|
90
|
+
AWS_ARGS=()
|
|
91
|
+
[ -n "$AWS_PROFILE_OVERRIDE" ] && AWS_ARGS+=(--profile "$AWS_PROFILE_OVERRIDE")
|
|
92
|
+
AWS_ARGS+=(--region "$AWS_REGION_OVERRIDE")
|
|
93
|
+
|
|
94
|
+
# Tmp dirs.
|
|
95
|
+
BASELINE="$(mktemp -d -t hq-rebaseline-baseline-XXXXXX)"
|
|
96
|
+
STAGING="$(mktemp -d -t hq-rebaseline-stage-XXXXXX)"
|
|
97
|
+
|
|
98
|
+
cleanup() {
|
|
99
|
+
local rc=$?
|
|
100
|
+
if [ "$KEEP_TMP" = "1" ]; then
|
|
101
|
+
echo "==> --keep-tmp set; preserving $BASELINE and $STAGING" >&2
|
|
102
|
+
else
|
|
103
|
+
rm -rf "${BASELINE:-}" "${STAGING:-}"
|
|
104
|
+
fi
|
|
105
|
+
return $rc
|
|
106
|
+
}
|
|
107
|
+
trap cleanup EXIT
|
|
108
|
+
|
|
109
|
+
cat <<EOF
|
|
110
|
+
==> vault-rebaseline
|
|
111
|
+
prs: $PRS
|
|
112
|
+
bucket: s3://$BUCKET
|
|
113
|
+
baseline: $SOURCE_REPO @ $REF (version $VERSION)
|
|
114
|
+
baseline tmp: $BASELINE
|
|
115
|
+
staging tmp: $STAGING
|
|
116
|
+
dry-run: $([ "$DRY_RUN" = "1" ] && echo "ON" || echo "off")
|
|
117
|
+
delete v14: $([ "$DELETE_CRUFT" = "1" ] && echo "ON" || echo "off")
|
|
118
|
+
keep-tmp: $([ "$KEEP_TMP" = "1" ] && echo "yes" || echo "no")
|
|
119
|
+
EOF
|
|
120
|
+
|
|
121
|
+
# ---------- phase 1: clone baseline ----------
|
|
122
|
+
echo "" >&2
|
|
123
|
+
echo "==> [1/3] clone $SOURCE_REPO@$REF (depth=1)" >&2
|
|
124
|
+
git clone --quiet --depth 1 --branch "$REF" "https://github.com/$SOURCE_REPO.git" "$BASELINE" 2>&1 | tail -3 || {
|
|
125
|
+
echo "error: clone failed. Is $SOURCE_REPO@$REF a valid ref?" >&2
|
|
126
|
+
exit 1
|
|
127
|
+
}
|
|
128
|
+
echo "==> clone done. baseline files: $(find "$BASELINE" -type f | wc -l | tr -d ' ') (excl .git)" >&2
|
|
129
|
+
|
|
130
|
+
# ---------- phase 2: stage with symlink serialization ----------
|
|
131
|
+
echo "" >&2
|
|
132
|
+
echo "==> [2/3] stage baseline tree (serialize symlinks → hq-symlink: markers)" >&2
|
|
133
|
+
|
|
134
|
+
n_files=0; n_syms=0
|
|
135
|
+
SYMLIST="$STAGING.symlinks.txt"
|
|
136
|
+
: > "$SYMLIST"
|
|
137
|
+
|
|
138
|
+
while IFS= read -r -d '' src; do
|
|
139
|
+
rel="${src#"$BASELINE/"}"
|
|
140
|
+
[ "$rel" = "$src" ] && continue
|
|
141
|
+
[ -z "$rel" ] && continue
|
|
142
|
+
|
|
143
|
+
# Skip the clone's .git directory (we're shipping just the tree, not the repo).
|
|
144
|
+
case "$rel" in
|
|
145
|
+
.git|.git/*) continue ;;
|
|
146
|
+
esac
|
|
147
|
+
|
|
148
|
+
dest="$STAGING/$rel"
|
|
149
|
+
mkdir -p "$(dirname "$dest")"
|
|
150
|
+
|
|
151
|
+
if [ -L "$src" ]; then
|
|
152
|
+
# Read the symlink's literal target string (apps/hq-cloud/src/s3.ts:417).
|
|
153
|
+
target="$(readlink "$src")"
|
|
154
|
+
printf 'hq-symlink:%s' "$target" > "$dest"
|
|
155
|
+
printf '%s\n' "$rel" >> "$SYMLIST"
|
|
156
|
+
n_syms=$((n_syms + 1))
|
|
157
|
+
elif [ -f "$src" ]; then
|
|
158
|
+
cp -p "$src" "$dest"
|
|
159
|
+
n_files=$((n_files + 1))
|
|
160
|
+
fi
|
|
161
|
+
done < <(find "$BASELINE" \( -type d -name .git -prune \) -o \( \( -type f -o -type l \) -print0 \))
|
|
162
|
+
|
|
163
|
+
echo " staged: $n_files files, $n_syms symlinks" >&2
|
|
164
|
+
|
|
165
|
+
# Strip files from staging that are KNOWN user-state — baseline ships
|
|
166
|
+
# default values for these but the vault almost certainly has the user's
|
|
167
|
+
# own customizations. Overwriting them would clobber config that lives
|
|
168
|
+
# nowhere else (the menubar's hq-cloud/ignore.ts excludes some of these
|
|
169
|
+
# from cloud sync, but earlier vault versions uploaded them).
|
|
170
|
+
#
|
|
171
|
+
# Each entry is a relative path inside the staging tree. Removed BEFORE
|
|
172
|
+
# `aws s3 sync` runs so the file is not even a candidate for upload.
|
|
173
|
+
PRESERVE_USER_STATE=(
|
|
174
|
+
# Company registry — user-owned, registers their connected companies'
|
|
175
|
+
# cloud_uid + bucket_name. v15 baseline ships an empty stub.
|
|
176
|
+
"companies/manifest.yaml"
|
|
177
|
+
# Claude Code prefs — user-customized model, env vars, hooks. v15 ships
|
|
178
|
+
# opinionated defaults that would overwrite e.g. MAX_THINKING_TOKENS,
|
|
179
|
+
# CLAUDE_CODE_SUBAGENT_MODEL, hook routing.
|
|
180
|
+
".claude/settings.json"
|
|
181
|
+
# Obsidian UI state — pane layout, bookmarks, graph view. v15 ships
|
|
182
|
+
# defaults; users with active Obsidian have their own.
|
|
183
|
+
".obsidian/workspace.json"
|
|
184
|
+
".obsidian/graph.json"
|
|
185
|
+
".obsidian/bookmarks.json"
|
|
186
|
+
".obsidian/app.json"
|
|
187
|
+
".obsidian/appearance.json"
|
|
188
|
+
".obsidian/core-plugins.json"
|
|
189
|
+
)
|
|
190
|
+
n_preserved=0
|
|
191
|
+
for rel in "${PRESERVE_USER_STATE[@]}"; do
|
|
192
|
+
if [ -f "$STAGING/$rel" ]; then
|
|
193
|
+
rm -f "$STAGING/$rel"
|
|
194
|
+
n_preserved=$((n_preserved + 1))
|
|
195
|
+
echo " preserving user state (skipped from upload): $rel" >&2
|
|
196
|
+
fi
|
|
197
|
+
done
|
|
198
|
+
|
|
199
|
+
# ---------- phase 3: push ----------
|
|
200
|
+
echo "" >&2
|
|
201
|
+
echo "==> [3/3] aws s3 sync $STAGING/ → s3://$BUCKET/" >&2
|
|
202
|
+
|
|
203
|
+
# `--size-only` because staging mtime is fresh and S3 LastModified reflects
|
|
204
|
+
# original upload time. Default mtime-based comparison would re-upload every
|
|
205
|
+
# content-stable file (same problem as vault-rescue.sh's push phase).
|
|
206
|
+
#
|
|
207
|
+
# No `--delete` by default. v14 baseline files coexist with v15 in S3 until
|
|
208
|
+
# either the user upgrades locally (menubar Update HQ) or --delete-v14-cruft
|
|
209
|
+
# is passed (handled below as a separate rm pass for a narrow allowlist).
|
|
210
|
+
SYNC_ARGS=("s3" "sync" "$STAGING/" "s3://$BUCKET/" "--size-only")
|
|
211
|
+
[ "$DRY_RUN" = "1" ] && SYNC_ARGS+=("--dryrun")
|
|
212
|
+
|
|
213
|
+
aws "${AWS_ARGS[@]}" "${SYNC_ARGS[@]}"
|
|
214
|
+
|
|
215
|
+
# Stamp the hq-symlink-target metadata header on every marker so the menubar
|
|
216
|
+
# pull materializes them as real symlinks (apps/hq-cloud/src/s3.ts:466-468
|
|
217
|
+
# prefers header detection; body sniff is fallback).
|
|
218
|
+
n_stamped=0
|
|
219
|
+
if [ "$DRY_RUN" = "0" ] && [ -s "$SYMLIST" ]; then
|
|
220
|
+
echo "" >&2
|
|
221
|
+
echo " stamping hq-symlink-target metadata on $(wc -l < "$SYMLIST" | tr -d ' ') markers" >&2
|
|
222
|
+
while IFS= read -r rel; do
|
|
223
|
+
[ -z "$rel" ] && continue
|
|
224
|
+
aws "${AWS_ARGS[@]}" s3api put-object \
|
|
225
|
+
--bucket "$BUCKET" \
|
|
226
|
+
--key "$rel" \
|
|
227
|
+
--body "$STAGING/$rel" \
|
|
228
|
+
--metadata "hq-symlink-target=1" \
|
|
229
|
+
--content-type "text/plain" \
|
|
230
|
+
>/dev/null
|
|
231
|
+
n_stamped=$((n_stamped + 1))
|
|
232
|
+
done < "$SYMLIST"
|
|
233
|
+
echo " stamped: $n_stamped" >&2
|
|
234
|
+
fi
|
|
235
|
+
|
|
236
|
+
# Optional: delete v14-era root files that don't exist in v15. Narrow
|
|
237
|
+
# allowlist — files we can confidently say were always release-shipped and
|
|
238
|
+
# would never be user-customized. Everything else stays.
|
|
239
|
+
n_deleted=0
|
|
240
|
+
if [ "$DELETE_CRUFT" = "1" ]; then
|
|
241
|
+
echo "" >&2
|
|
242
|
+
echo " deleting v14 root-file cruft" >&2
|
|
243
|
+
V14_CRUFT=(
|
|
244
|
+
"CHANGELOG.md"
|
|
245
|
+
"README.md"
|
|
246
|
+
"LICENSE"
|
|
247
|
+
"setup.sh"
|
|
248
|
+
"RELEASE-NOTES-v14.0.0.md"
|
|
249
|
+
"MIGRATION-v11.md"
|
|
250
|
+
"USER-GUIDE.md"
|
|
251
|
+
"CONTRIBUTING.md"
|
|
252
|
+
)
|
|
253
|
+
for key in "${V14_CRUFT[@]}"; do
|
|
254
|
+
# Probe first — only delete if it actually exists.
|
|
255
|
+
if aws "${AWS_ARGS[@]}" s3api head-object --bucket "$BUCKET" --key "$key" >/dev/null 2>&1; then
|
|
256
|
+
if [ "$DRY_RUN" = "1" ]; then
|
|
257
|
+
echo " would delete: $key" >&2
|
|
258
|
+
else
|
|
259
|
+
aws "${AWS_ARGS[@]}" s3api delete-object --bucket "$BUCKET" --key "$key" >/dev/null
|
|
260
|
+
echo " deleted: $key" >&2
|
|
261
|
+
fi
|
|
262
|
+
n_deleted=$((n_deleted + 1))
|
|
263
|
+
fi
|
|
264
|
+
done
|
|
265
|
+
fi
|
|
266
|
+
|
|
267
|
+
echo "" >&2
|
|
268
|
+
echo "==> vault-rebaseline complete." >&2
|
|
269
|
+
echo " bucket: s3://$BUCKET" >&2
|
|
270
|
+
echo " version: $VERSION ($SOURCE_REPO@$REF)" >&2
|
|
271
|
+
echo " pushed: $n_files files, $n_syms symlinks (gross — preserved excluded)" >&2
|
|
272
|
+
echo " user-state preserved: $n_preserved files" >&2
|
|
273
|
+
echo " stamped: $n_stamped" >&2
|
|
274
|
+
echo " deleted: $n_deleted (v14 cruft)" >&2
|
|
275
|
+
exit 0
|
package/scripts/vault-rescue.sh
CHANGED
|
@@ -170,6 +170,14 @@ RESCUE_ARGS=(
|
|
|
170
170
|
--no-backup
|
|
171
171
|
--yes
|
|
172
172
|
)
|
|
173
|
+
# --no-backup is intentional: replace-rescue.sh's own pre-update snapshot
|
|
174
|
+
# would land at ~/.hq/backups/ on the OPERATOR's machine, not the vault
|
|
175
|
+
# owner's — so it doesn't help the user we're rescuing. Phase 1 already
|
|
176
|
+
# leaves the pulled tmp tree on disk (preserved with --keep-tmp), which IS
|
|
177
|
+
# the meaningful pre-rescue snapshot. Operators who want a hardened
|
|
178
|
+
# backup tarball it themselves before invoking this script.
|
|
179
|
+
# Requires hq-sync ≥0.6.4 (indigoai-us/hq-sync#170 — earlier versions
|
|
180
|
+
# return exit 1 on the --no-backup branch and kill us mid-flight).
|
|
173
181
|
[ "$DRY_RUN" = "1" ] && RESCUE_ARGS+=(--dry-run)
|
|
174
182
|
[ -n "$FLOOR_SHA" ] && RESCUE_ARGS+=(--floor-sha "$FLOOR_SHA")
|
|
175
183
|
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
runRunnerWithLoop,
|
|
17
17
|
resolveDeletePolicy,
|
|
18
18
|
resolveSkipPersonal,
|
|
19
|
+
resolvePresignTransport,
|
|
19
20
|
routeChangeToTarget,
|
|
20
21
|
buildTargetedPushArgv,
|
|
21
22
|
resolvePullScope,
|
|
@@ -3126,6 +3127,46 @@ describe("resolveSkipPersonal", () => {
|
|
|
3126
3127
|
);
|
|
3127
3128
|
});
|
|
3128
3129
|
|
|
3130
|
+
// ---------------------------------------------------------------------------
|
|
3131
|
+
// resolvePresignTransport — @getindigo.ai rollout gate + env override
|
|
3132
|
+
// ---------------------------------------------------------------------------
|
|
3133
|
+
|
|
3134
|
+
describe("resolvePresignTransport", () => {
|
|
3135
|
+
it("ON for @getindigo.ai emails (case-insensitive), no override", () => {
|
|
3136
|
+
expect(resolvePresignTransport("me@getindigo.ai", undefined)).toBe(true);
|
|
3137
|
+
expect(resolvePresignTransport("ME@GetIndigo.AI", undefined)).toBe(true);
|
|
3138
|
+
});
|
|
3139
|
+
|
|
3140
|
+
it("OFF for non-getindigo emails, no override", () => {
|
|
3141
|
+
expect(resolvePresignTransport("me@example.com", undefined)).toBe(false);
|
|
3142
|
+
expect(resolvePresignTransport(undefined, undefined)).toBe(false);
|
|
3143
|
+
// A lookalike domain must not match the endsWith check loosely.
|
|
3144
|
+
expect(resolvePresignTransport("me@notgetindigo.ai.evil.com", undefined)).toBe(
|
|
3145
|
+
false,
|
|
3146
|
+
);
|
|
3147
|
+
});
|
|
3148
|
+
|
|
3149
|
+
it.each(["1", "true", "yes", "on", "ON", " True "])(
|
|
3150
|
+
"override '%s' forces ON even for a non-getindigo email",
|
|
3151
|
+
(val) => {
|
|
3152
|
+
expect(resolvePresignTransport("me@example.com", val)).toBe(true);
|
|
3153
|
+
},
|
|
3154
|
+
);
|
|
3155
|
+
|
|
3156
|
+
it.each(["0", "false", "no", "off", "OFF"])(
|
|
3157
|
+
"override '%s' forces OFF even for a getindigo email",
|
|
3158
|
+
(val) => {
|
|
3159
|
+
expect(resolvePresignTransport("me@getindigo.ai", val)).toBe(false);
|
|
3160
|
+
},
|
|
3161
|
+
);
|
|
3162
|
+
|
|
3163
|
+
it("blank/unrecognized override falls through to the email check", () => {
|
|
3164
|
+
expect(resolvePresignTransport("me@getindigo.ai", "")).toBe(true);
|
|
3165
|
+
expect(resolvePresignTransport("me@getindigo.ai", "maybe")).toBe(true);
|
|
3166
|
+
expect(resolvePresignTransport("me@example.com", "maybe")).toBe(false);
|
|
3167
|
+
});
|
|
3168
|
+
});
|
|
3169
|
+
|
|
3129
3170
|
// ---------------------------------------------------------------------------
|
|
3130
3171
|
// resolvePullScope (US-005) — effective download scope per company leg
|
|
3131
3172
|
// ---------------------------------------------------------------------------
|
package/src/bin/sync-runner.ts
CHANGED
|
@@ -91,6 +91,11 @@ import { share as defaultShare } from "../cli/share.js";
|
|
|
91
91
|
import type { ShareOptions, ShareResult } from "../cli/share.js";
|
|
92
92
|
import type { ConflictStrategy } from "../cli/conflict.js";
|
|
93
93
|
import type { UploadAuthor } from "../s3.js";
|
|
94
|
+
import {
|
|
95
|
+
setObjectIOFactory,
|
|
96
|
+
presignObjectIOFactory,
|
|
97
|
+
type PresignTransportClient,
|
|
98
|
+
} from "../object-io.js";
|
|
94
99
|
import { collectAndSendTelemetry } from "../telemetry.js";
|
|
95
100
|
import { collectAndSendSkillTelemetry } from "../skill-telemetry.js";
|
|
96
101
|
import { reindexAfterSync } from "../qmd-reindex.js";
|
|
@@ -204,6 +209,27 @@ export function resolveSkipPersonal(flag: boolean): boolean {
|
|
|
204
209
|
return env === "1" || env === "true" || env === "yes";
|
|
205
210
|
}
|
|
206
211
|
|
|
212
|
+
/**
|
|
213
|
+
* Decide whether this session uses the presigned-URL transport.
|
|
214
|
+
*
|
|
215
|
+
* Rollout gate: ON for accounts whose verified email is `@getindigo.ai`.
|
|
216
|
+
* `HQ_SYNC_PRESIGN_TRANSPORT` overrides the email check in both directions
|
|
217
|
+
* (`1`/`true`/`yes`/`on` → force on, `0`/`false`/`no`/`off` → force off) so
|
|
218
|
+
* the transport can be exercised by non-getindigo testers or rolled back for
|
|
219
|
+
* getindigo accounts without a redeploy. An unset/blank override falls through
|
|
220
|
+
* to the email check; an unrecognized override value is ignored (email check
|
|
221
|
+
* wins) rather than silently forcing a state.
|
|
222
|
+
*/
|
|
223
|
+
export function resolvePresignTransport(
|
|
224
|
+
email: string | undefined,
|
|
225
|
+
override: string | undefined,
|
|
226
|
+
): boolean {
|
|
227
|
+
const o = (override ?? "").trim().toLowerCase();
|
|
228
|
+
if (o === "1" || o === "true" || o === "yes" || o === "on") return true;
|
|
229
|
+
if (o === "0" || o === "false" || o === "no" || o === "off") return false;
|
|
230
|
+
return typeof email === "string" && email.toLowerCase().endsWith("@getindigo.ai");
|
|
231
|
+
}
|
|
232
|
+
|
|
207
233
|
// Personal-vault scope (exclusion list + path computer) lives in
|
|
208
234
|
// `../personal-vault.ts` so the `hq sync` CLI and this runner share the same
|
|
209
235
|
// rules. Re-exported here for back-compat with any callers still importing
|
|
@@ -953,6 +979,32 @@ export async function runRunner(
|
|
|
953
979
|
? { userSub: claims.sub, email: claims.email }
|
|
954
980
|
: undefined;
|
|
955
981
|
|
|
982
|
+
// ---- transport selection (presigned-URL vs STS-direct-S3) -------------
|
|
983
|
+
// Default transport is unchanged: STS-vended credentials + direct S3 SDK.
|
|
984
|
+
// The presigned-URL transport (vault `list` + `presign` endpoints) is
|
|
985
|
+
// feature-flagged to @getindigo.ai accounts during rollout. The gate runs
|
|
986
|
+
// ONCE here — every s3.ts call in this session's fanout resolves through
|
|
987
|
+
// the installed factory. `HQ_SYNC_PRESIGN_TRANSPORT` is an explicit
|
|
988
|
+
// override (1/true → force on, 0/false → force off) that wins over the
|
|
989
|
+
// email check, so the flag can be flipped for testing or rollback without
|
|
990
|
+
// a redeploy. Setting the factory unconditionally (even to the default)
|
|
991
|
+
// keeps the choice deterministic if a prior run mutated module state.
|
|
992
|
+
const presignCapable = client as Partial<PresignTransportClient>;
|
|
993
|
+
if (
|
|
994
|
+
resolvePresignTransport(
|
|
995
|
+
claims?.email,
|
|
996
|
+
process.env.HQ_SYNC_PRESIGN_TRANSPORT,
|
|
997
|
+
) &&
|
|
998
|
+
typeof presignCapable.presign === "function" &&
|
|
999
|
+
typeof presignCapable.listFiles === "function"
|
|
1000
|
+
) {
|
|
1001
|
+
setObjectIOFactory(
|
|
1002
|
+
presignObjectIOFactory(presignCapable as PresignTransportClient),
|
|
1003
|
+
);
|
|
1004
|
+
} else {
|
|
1005
|
+
setObjectIOFactory(null);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
956
1008
|
// ---- resolve targets --------------------------------------------------
|
|
957
1009
|
let memberships: Pick<Membership, "companyUid">[];
|
|
958
1010
|
try {
|
package/src/cli/share.test.ts
CHANGED
|
@@ -21,6 +21,8 @@ vi.mock("../s3.js", () => ({
|
|
|
21
21
|
listRemoteFiles: vi.fn().mockResolvedValue([]),
|
|
22
22
|
deleteRemoteFile: vi.fn().mockResolvedValue(undefined),
|
|
23
23
|
headRemoteFile: vi.fn().mockResolvedValue(null),
|
|
24
|
+
primeObjectTransport: vi.fn().mockResolvedValue(undefined),
|
|
25
|
+
primeUploads: vi.fn().mockResolvedValue(undefined),
|
|
24
26
|
}));
|
|
25
27
|
|
|
26
28
|
// Mock readline so the interactive-conflict serialization test can observe
|
package/src/cli/share.ts
CHANGED
|
@@ -9,7 +9,15 @@ import * as fs from "fs";
|
|
|
9
9
|
import * as path from "path";
|
|
10
10
|
import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
|
|
11
11
|
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
uploadFile,
|
|
14
|
+
uploadSymlink,
|
|
15
|
+
headRemoteFile,
|
|
16
|
+
deleteRemoteFile,
|
|
17
|
+
downloadFile,
|
|
18
|
+
primeObjectTransport,
|
|
19
|
+
primeUploads,
|
|
20
|
+
} from "../s3.js";
|
|
13
21
|
import * as crypto from "crypto";
|
|
14
22
|
import type { UploadAuthor } from "../s3.js";
|
|
15
23
|
import {
|
|
@@ -784,6 +792,21 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
784
792
|
uploadItems.push(item);
|
|
785
793
|
}
|
|
786
794
|
|
|
795
|
+
// Batch pre-mint PUT URLs (+ the created-at HEADs) for the whole upload set,
|
|
796
|
+
// signing the SAME metadata the pool below computes so each task replays the
|
|
797
|
+
// cached headers and skips its own presign. Turns an N-file push from ~N
|
|
798
|
+
// presign calls into ceil(N/1000) GET + ceil(N/1000) PUT — keeping a bulk
|
|
799
|
+
// push under the 100/hr limit. No-op on the S3 SDK transport; best-effort.
|
|
800
|
+
await primeUploads(
|
|
801
|
+
ctx,
|
|
802
|
+
uploadItems.map((it) => ({
|
|
803
|
+
key: it.relativePath,
|
|
804
|
+
localPath: it.absolutePath,
|
|
805
|
+
isSymlink: it.kind === "symlink",
|
|
806
|
+
author: options.author,
|
|
807
|
+
})),
|
|
808
|
+
);
|
|
809
|
+
|
|
787
810
|
// Phase B: parallel upload pool. Each task runs the full per-item flow
|
|
788
811
|
// (HEAD + conflict + PUT + journal stamp + emit). Aborts flip the
|
|
789
812
|
// shared `aborted` flag and the pool stops draining the queue; tasks
|
|
@@ -1100,7 +1123,11 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1100
1123
|
// leg of `sync now` re-pulls naturally via the existing
|
|
1101
1124
|
// `hasRemoteChanged` path. Emit a dedicated event so UIs can
|
|
1102
1125
|
// surface the refusal without inferring it from absence.
|
|
1103
|
-
|
|
1126
|
+
// Batch pre-mint DELETE URLs so a large delete set is ~ceil(N/1000) presign
|
|
1127
|
+
// calls, not N. No-op on the S3 SDK transport; best-effort.
|
|
1128
|
+
const deleteKeys = [...deletePlan.toDelete, ...decommissionPlan];
|
|
1129
|
+
await primeObjectTransport(ctx, "delete", deleteKeys);
|
|
1130
|
+
for (const relativePath of deleteKeys) {
|
|
1104
1131
|
if (vaultConfig && isExpiringSoon(ctx.expiresAt)) {
|
|
1105
1132
|
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
1106
1133
|
}
|