@indigoai-us/hq-cloud 6.15.0 → 6.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-mutation.d.ts +16 -0
- package/dist/bin/sync-mutation.d.ts.map +1 -0
- package/dist/bin/sync-mutation.js +60 -0
- package/dist/bin/sync-mutation.js.map +1 -0
- package/dist/bin/sync-mutation.test.d.ts +2 -0
- package/dist/bin/sync-mutation.test.d.ts.map +1 -0
- package/dist/bin/sync-mutation.test.js +165 -0
- package/dist/bin/sync-mutation.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +8 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +16 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.d.ts +2 -0
- package/dist/bin/sync-runner-company.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.test.js +36 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +98 -8
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +17 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +109 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/conflict-recovery.test.d.ts +2 -0
- package/dist/cli/conflict-recovery.test.d.ts.map +1 -0
- package/dist/cli/conflict-recovery.test.js +201 -0
- package/dist/cli/conflict-recovery.test.js.map +1 -0
- package/dist/cli/conflict.d.ts +60 -0
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +333 -0
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/sync.d.ts +35 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +100 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +85 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/skill-telemetry.d.ts +6 -0
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +14 -2
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +79 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/candidate-uploader.d.ts +88 -0
- package/dist/sync/candidate-uploader.d.ts.map +1 -0
- package/dist/sync/candidate-uploader.js +212 -0
- package/dist/sync/candidate-uploader.js.map +1 -0
- package/dist/sync/candidate-uploader.test.d.ts +2 -0
- package/dist/sync/candidate-uploader.test.d.ts.map +1 -0
- package/dist/sync/candidate-uploader.test.js +132 -0
- package/dist/sync/candidate-uploader.test.js.map +1 -0
- package/dist/sync/delta-client.d.ts +73 -0
- package/dist/sync/delta-client.d.ts.map +1 -0
- package/dist/sync/delta-client.js +201 -0
- package/dist/sync/delta-client.js.map +1 -0
- package/dist/sync/delta-client.test.d.ts +2 -0
- package/dist/sync/delta-client.test.d.ts.map +1 -0
- package/dist/sync/delta-client.test.js +97 -0
- package/dist/sync/delta-client.test.js.map +1 -0
- package/dist/sync/durable-apply.d.ts +76 -0
- package/dist/sync/durable-apply.d.ts.map +1 -0
- package/dist/sync/durable-apply.js +530 -0
- package/dist/sync/durable-apply.js.map +1 -0
- package/dist/sync/durable-apply.test.d.ts +2 -0
- package/dist/sync/durable-apply.test.d.ts.map +1 -0
- package/dist/sync/durable-apply.test.js +180 -0
- package/dist/sync/durable-apply.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts +33 -1
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +149 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +142 -1
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/index.d.ts +2 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/multipart-uploader.d.ts +99 -0
- package/dist/sync/multipart-uploader.d.ts.map +1 -0
- package/dist/sync/multipart-uploader.js +447 -0
- package/dist/sync/multipart-uploader.js.map +1 -0
- package/dist/sync/multipart-uploader.test.d.ts +2 -0
- package/dist/sync/multipart-uploader.test.d.ts.map +1 -0
- package/dist/sync/multipart-uploader.test.js +119 -0
- package/dist/sync/multipart-uploader.test.js.map +1 -0
- package/dist/sync/mutation-client.d.ts +85 -0
- package/dist/sync/mutation-client.d.ts.map +1 -0
- package/dist/sync/mutation-client.js +245 -0
- package/dist/sync/mutation-client.js.map +1 -0
- package/dist/sync/mutation-client.test.d.ts +2 -0
- package/dist/sync/mutation-client.test.d.ts.map +1 -0
- package/dist/sync/mutation-client.test.js +51 -0
- package/dist/sync/mutation-client.test.js.map +1 -0
- package/dist/sync/push-receiver.d.ts +45 -0
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +101 -0
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +54 -2
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/scope-inventory-client.d.ts +69 -0
- package/dist/sync/scope-inventory-client.d.ts.map +1 -0
- package/dist/sync/scope-inventory-client.js +210 -0
- package/dist/sync/scope-inventory-client.js.map +1 -0
- package/dist/sync/scope-inventory-client.test.d.ts +2 -0
- package/dist/sync/scope-inventory-client.test.d.ts.map +1 -0
- package/dist/sync/scope-inventory-client.test.js +94 -0
- package/dist/sync/scope-inventory-client.test.js.map +1 -0
- package/dist/sync/snapshot-client.d.ts +98 -0
- package/dist/sync/snapshot-client.d.ts.map +1 -0
- package/dist/sync/snapshot-client.js +402 -0
- package/dist/sync/snapshot-client.js.map +1 -0
- package/dist/sync/snapshot-client.test.d.ts +2 -0
- package/dist/sync/snapshot-client.test.d.ts.map +1 -0
- package/dist/sync/snapshot-client.test.js +169 -0
- package/dist/sync/snapshot-client.test.js.map +1 -0
- package/dist/sync/uploader-finalization.d.ts +97 -0
- package/dist/sync/uploader-finalization.d.ts.map +1 -0
- package/dist/sync/uploader-finalization.js +273 -0
- package/dist/sync/uploader-finalization.js.map +1 -0
- package/dist/sync/uploader-finalization.test.d.ts +2 -0
- package/dist/sync/uploader-finalization.test.d.ts.map +1 -0
- package/dist/sync/uploader-finalization.test.js +92 -0
- package/dist/sync/uploader-finalization.test.js.map +1 -0
- package/dist/telemetry.d.ts +11 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +21 -2
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +80 -0
- package/dist/telemetry.test.js.map +1 -1
- package/package.json +6 -1
- package/.claude/policies/hq-cloud-esm-cannot-spy-fs-builtins.md +0 -30
- package/.claude/policies/hq-cloud-strip-types-no-parameter-properties.md +0 -22
- package/.github/workflows/ci.yml +0 -84
- package/.github/workflows/publish.yml +0 -56
- package/.github/workflows/unreleased-commits-nag.yml +0 -256
- package/eslint.config.js +0 -67
- package/pnpm-workspace.yaml +0 -2
- package/scripts/presign-transport-e2e.mjs +0 -250
- package/scripts/vault-rebaseline.sh +0 -323
- package/scripts/vault-rescue.sh +0 -332
- package/src/active-company.test.ts +0 -188
- package/src/active-company.ts +0 -168
- package/src/agent-codex-instructions.test.ts +0 -332
- package/src/agent-codex-instructions.ts +0 -309
- package/src/auth.ts +0 -146
- package/src/backup-prune.test.ts +0 -98
- package/src/backup-prune.ts +0 -182
- package/src/bin/backup-prune-runner.ts +0 -33
- package/src/bin/rescue-runner.ts +0 -25
- package/src/bin/sync-runner-company.ts +0 -695
- package/src/bin/sync-runner-events.test.ts +0 -143
- package/src/bin/sync-runner-events.ts +0 -55
- package/src/bin/sync-runner-planning.test.ts +0 -311
- package/src/bin/sync-runner-planning.ts +0 -258
- package/src/bin/sync-runner-rollup.test.ts +0 -37
- package/src/bin/sync-runner-rollup.ts +0 -97
- package/src/bin/sync-runner-telemetry.ts +0 -15
- package/src/bin/sync-runner-watch-loop.ts +0 -1235
- package/src/bin/sync-runner-watch-routes.test.ts +0 -71
- package/src/bin/sync-runner-watch-routes.ts +0 -184
- package/src/bin/sync-runner.test.ts +0 -8767
- package/src/bin/sync-runner.ts +0 -2190
- package/src/cli/accept.ts +0 -124
- package/src/cli/conflict.ts +0 -119
- package/src/cli/doctor.test.ts +0 -581
- package/src/cli/doctor.ts +0 -642
- package/src/cli/index.ts +0 -49
- package/src/cli/invite.test.ts +0 -250
- package/src/cli/invite.ts +0 -214
- package/src/cli/promote.ts +0 -157
- package/src/cli/reindex-knowledge.test.ts +0 -307
- package/src/cli/reindex-knowledge.ts +0 -450
- package/src/cli/reindex.test.ts +0 -957
- package/src/cli/reindex.ts +0 -979
- package/src/cli/rescue-classify-ordering.test.ts +0 -548
- package/src/cli/rescue-clone-diagnostics.test.ts +0 -120
- package/src/cli/rescue-core.ts +0 -3011
- package/src/cli/rescue-drift-reconcile.test.ts +0 -179
- package/src/cli/rescue-drop-dir-symlink.test.ts +0 -224
- package/src/cli/rescue-exec-bit-preserve.test.ts +0 -187
- package/src/cli/rescue-hq-root-guard.test.ts +0 -232
- package/src/cli/rescue-journal-reconcile.test.ts +0 -215
- package/src/cli/rescue-mtime-preserve.test.ts +0 -203
- package/src/cli/rescue-settings-reconcile.test.ts +0 -637
- package/src/cli/rescue-snapshot.test.ts +0 -57
- package/src/cli/rescue-snapshot.ts +0 -51
- package/src/cli/rescue.reindex.test.ts +0 -63
- package/src/cli/rescue.test.ts +0 -131
- package/src/cli/rescue.ts +0 -182
- package/src/cli/share.test.ts +0 -7843
- package/src/cli/share.ts +0 -3663
- package/src/cli/sync-scope.test.ts +0 -652
- package/src/cli/sync.test.ts +0 -5207
- package/src/cli/sync.ts +0 -3470
- package/src/cli/tombstones.ts +0 -106
- package/src/cli/watch-event-push-conflict.test.ts +0 -234
- package/src/client-info.test.ts +0 -214
- package/src/client-info.ts +0 -121
- package/src/cognito-auth.test.ts +0 -712
- package/src/cognito-auth.ts +0 -1422
- package/src/company-resolver.test.ts +0 -618
- package/src/company-resolver.ts +0 -521
- package/src/context.test.ts +0 -583
- package/src/context.ts +0 -378
- package/src/daemon-worker.ts +0 -26
- package/src/daemon.ts +0 -99
- package/src/entity-resolver.test.ts +0 -315
- package/src/entity-resolver.ts +0 -180
- package/src/ignore.test.ts +0 -466
- package/src/ignore.ts +0 -469
- package/src/index.ts +0 -439
- package/src/journal.test.ts +0 -968
- package/src/journal.ts +0 -765
- package/src/lib/cloud-authoritative.test.ts +0 -45
- package/src/lib/cloud-authoritative.ts +0 -59
- package/src/lib/conflict-file.ts +0 -86
- package/src/lib/conflict-index.ts +0 -289
- package/src/lib/conflict.test.ts +0 -348
- package/src/lib/describe-error.test.ts +0 -100
- package/src/lib/describe-error.ts +0 -58
- package/src/lib/exit-codes.ts +0 -24
- package/src/lib/machine-id.test.ts +0 -231
- package/src/lib/machine-id.ts +0 -175
- package/src/lib/net-errors.test.ts +0 -65
- package/src/lib/net-errors.ts +0 -86
- package/src/lib/readlink-safe.test.ts +0 -43
- package/src/lib/readlink-safe.ts +0 -29
- package/src/local-path-codec.test.ts +0 -138
- package/src/local-path-codec.ts +0 -161
- package/src/machine-auth.test.ts +0 -1323
- package/src/manifest-reconcile.test.ts +0 -1123
- package/src/manifest-reconcile.ts +0 -518
- package/src/object-io.test.ts +0 -1221
- package/src/object-io.ts +0 -1306
- package/src/operation-lock.test.ts +0 -484
- package/src/operation-lock.ts +0 -680
- package/src/outcome-telemetry.test.ts +0 -498
- package/src/outcome-telemetry.ts +0 -639
- package/src/personal-vault-exclusions.test.ts +0 -308
- package/src/personal-vault-exclusions.ts +0 -354
- package/src/personal-vault.test.ts +0 -756
- package/src/personal-vault.ts +0 -496
- package/src/prefix-coalesce.test.ts +0 -240
- package/src/prefix-coalesce.ts +0 -273
- package/src/public-surface.test.ts +0 -117
- package/src/qmd-reindex.test.ts +0 -877
- package/src/qmd-reindex.ts +0 -842
- package/src/read-only-state-dir.test.ts +0 -188
- package/src/remote-pull.test.ts +0 -1130
- package/src/remote-pull.ts +0 -618
- package/src/s3.symlink-materialize.test.ts +0 -492
- package/src/s3.test.ts +0 -1789
- package/src/s3.ts +0 -1532
- package/src/schemas/signal-types.test.ts +0 -82
- package/src/schemas/signal-types.ts +0 -38
- package/src/schemas/source-channels.test.ts +0 -82
- package/src/schemas/source-channels.ts +0 -53
- package/src/scope-shrink.test.ts +0 -633
- package/src/scope-shrink.ts +0 -481
- package/src/signals/get.test.ts +0 -310
- package/src/signals/get.ts +0 -75
- package/src/signals/internals.ts +0 -195
- package/src/signals/list.test.ts +0 -420
- package/src/signals/list.ts +0 -79
- package/src/signals/parse.ts +0 -8
- package/src/signals/types.ts +0 -91
- package/src/skill-telemetry.test.ts +0 -1825
- package/src/skill-telemetry.ts +0 -1439
- package/src/sources/get.test.ts +0 -293
- package/src/sources/get.ts +0 -66
- package/src/sources/internals.ts +0 -198
- package/src/sources/list.test.ts +0 -402
- package/src/sources/list.ts +0 -84
- package/src/sources/parse.ts +0 -43
- package/src/sources/types.ts +0 -84
- package/src/sync/event-sync.test.ts +0 -594
- package/src/sync/event-sync.ts +0 -545
- package/src/sync/feature-flags.test.ts +0 -378
- package/src/sync/feature-flags.ts +0 -62
- package/src/sync/index.ts +0 -76
- package/src/sync/lease-client.test.ts +0 -128
- package/src/sync/lease-client.ts +0 -207
- package/src/sync/logger.test.ts +0 -242
- package/src/sync/logger.ts +0 -79
- package/src/sync/metrics.test.ts +0 -462
- package/src/sync/metrics.ts +0 -213
- package/src/sync/pull-scope.ts +0 -265
- package/src/sync/push-event.test.ts +0 -266
- package/src/sync/push-event.ts +0 -224
- package/src/sync/push-receiver.test.ts +0 -566
- package/src/sync/push-receiver.ts +0 -1048
- package/src/sync/push-transport.ts +0 -231
- package/src/sync/realtime-rollout.test.ts +0 -86
- package/src/sync/realtime-rollout.ts +0 -262
- package/src/sync/state-store.test.ts +0 -194
- package/src/sync/state-store.ts +0 -727
- package/src/sync-core.ts +0 -58
- package/src/sync-progress.test.ts +0 -94
- package/src/sync-progress.ts +0 -140
- package/src/telemetry-events.test.ts +0 -88
- package/src/telemetry-events.ts +0 -205
- package/src/telemetry.test.ts +0 -1280
- package/src/telemetry.ts +0 -1109
- package/src/types.ts +0 -314
- package/src/vault-client.test.ts +0 -1380
- package/src/vault-client.ts +0 -1694
- package/src/version.ts +0 -24
- package/src/watch-roots.test.ts +0 -278
- package/src/watch-roots.ts +0 -162
- package/src/watcher-event-gate.test.ts +0 -212
- package/src/watcher.test.ts +0 -1079
- package/src/watcher.ts +0 -1741
- package/test/e2e/sync/cross-tenant-isolation.test.ts +0 -630
- package/test/e2e/sync/skill-telemetry-oversized-transcript.test.ts +0 -124
- package/test/e2e/sync/transient-company-leg.test.ts +0 -384
- package/test/e2e/sync/windows-unreadable-link-leg.test.ts +0 -191
- package/test/e2e/watcher-real-chokidar.test.ts +0 -165
- package/test/e2e/watcher-recursive-backend.test.ts +0 -181
- package/test/e2e/watcher-scoped-coverage.test.ts +0 -381
- package/test/invite-flow.integration.test.ts +0 -244
- package/test/joiner-manifest-reconcile.integration.test.ts +0 -322
- package/test/share-sync.integration.test.ts +0 -213
- package/tsconfig.json +0 -19
- package/vitest.config.ts +0 -22
package/src/s3.ts
DELETED
|
@@ -1,1532 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* S3 operations — upload, download, list, delete.
|
|
3
|
-
*
|
|
4
|
-
* VLT-5: All operations now accept an EntityContext (entity-aware bucket +
|
|
5
|
-
* STS-scoped credentials) instead of reading static env config. The caller
|
|
6
|
-
* is responsible for resolving the context via resolveEntityContext().
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import * as fs from "fs";
|
|
10
|
-
import * as path from "path";
|
|
11
|
-
import * as crypto from "crypto";
|
|
12
|
-
import { Readable, Transform } from "stream";
|
|
13
|
-
import { pipeline } from "stream/promises";
|
|
14
|
-
import type { EntityContext } from "./types.js";
|
|
15
|
-
import {
|
|
16
|
-
resolveObjectIO,
|
|
17
|
-
type GetObjectStreamResult,
|
|
18
|
-
type ObjectIO,
|
|
19
|
-
type PutPrecondition,
|
|
20
|
-
} from "./object-io.js";
|
|
21
|
-
|
|
22
|
-
// Byte/metadata transport is resolved per-call via resolveObjectIO(ctx) — the
|
|
23
|
-
// default is the AWS S3 SDK over STS-vended credentials (S3SdkObjectIO), but a
|
|
24
|
-
// session may select the presigned-URL transport (PresignObjectIO) via
|
|
25
|
-
// setObjectIOFactory. The symlink/mode/mtime/created-at semantics below are
|
|
26
|
-
// transport-agnostic: they compose on top of the ObjectIO primitives.
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Author identity stamped onto S3 user-defined metadata at upload time. The
|
|
30
|
-
* vault UI's "CREATED BY" column reads `Metadata['created-by']` back via
|
|
31
|
-
* HEAD; uploads without an author leave that column blank.
|
|
32
|
-
*/
|
|
33
|
-
export interface UploadAuthor {
|
|
34
|
-
/** Cognito sub — stable join key for per-member rollups. */
|
|
35
|
-
userSub: string;
|
|
36
|
-
/** Email for human display. */
|
|
37
|
-
email: string;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* S3 user metadata is ASCII-only (lowercased on read, capped at 2 KB total).
|
|
42
|
-
* Values that fail the printable-ASCII test or would push the keys over the
|
|
43
|
-
* cap are elided rather than throwing — partial attribution beats none. The
|
|
44
|
-
* shape mirrors `hq-console/src/lib/s3-vault.ts buildAuthorMetadata` so the
|
|
45
|
-
* read path on the consumer side stays a single check against
|
|
46
|
-
* `Metadata['created-by']`.
|
|
47
|
-
*/
|
|
48
|
-
function buildAuthorMetadata(
|
|
49
|
-
author: UploadAuthor,
|
|
50
|
-
createdAt: string,
|
|
51
|
-
): Record<string, string> {
|
|
52
|
-
const meta: Record<string, string> = {};
|
|
53
|
-
const sub = author.userSub.trim();
|
|
54
|
-
if (sub && /^[\x20-\x7E]+$/.test(sub)) {
|
|
55
|
-
meta["created-by-sub"] = sub;
|
|
56
|
-
}
|
|
57
|
-
const email = author.email.trim();
|
|
58
|
-
if (email && /^[\x20-\x7E]+$/.test(email)) {
|
|
59
|
-
meta["created-by"] = email;
|
|
60
|
-
}
|
|
61
|
-
if (createdAt && /^[\x20-\x7E]+$/.test(createdAt)) {
|
|
62
|
-
meta["created-at"] = createdAt;
|
|
63
|
-
}
|
|
64
|
-
return meta;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* S3 user-metadata header that marks an object as a symlink record.
|
|
69
|
-
* The value is now an OPAQUE MARKER ('1') — the target lives in the
|
|
70
|
-
* object body. Earlier drafts of this feature stored the target in
|
|
71
|
-
* metadata (raw, then base64), but S3 user-metadata is HTTP-header-
|
|
72
|
-
* bound: total ≤ 2 KiB across all user-defined keys + values. A
|
|
73
|
-
* sufficiently long POSIX target (or one with author-metadata
|
|
74
|
-
* adding to the total) would exceed the limit and PutObject would
|
|
75
|
-
* reject the upload outright. Moving the target to the body — which
|
|
76
|
-
* has no such limit — makes target length bounded only by S3's 5 GB
|
|
77
|
-
* object size cap. The metadata header still serves as the read-
|
|
78
|
-
* time discriminator (cheaper than peeking at body bytes via HEAD).
|
|
79
|
-
*
|
|
80
|
-
* Backward compat: downloadFile prefers the body (sliced after
|
|
81
|
-
* SYMLINK_BODY_PREFIX) for the target string. If the body doesn't
|
|
82
|
-
* carry the prefix (a legacy upload from earlier in this PR's
|
|
83
|
-
* lifetime), it falls back to base64-decoding the metadata value —
|
|
84
|
-
* the round-trip-validating decoder returns raw or decoded as
|
|
85
|
-
* appropriate. Any prior in-flight upload still resolves correctly.
|
|
86
|
-
*/
|
|
87
|
-
export const SYMLINK_TARGET_META_KEY = "hq-symlink-target";
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Constant value written to SYMLINK_TARGET_META_KEY. Any non-empty
|
|
91
|
-
* string would work as a discriminator — '1' is just compact and
|
|
92
|
-
* conventional for boolean flags in HTTP headers.
|
|
93
|
-
*/
|
|
94
|
-
export const SYMLINK_MARKER_META_VALUE = "1";
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* Encode a target for the S3 metadata header value. Retained as the
|
|
98
|
-
* legacy encoder so a downloader can still receive it and round-trip
|
|
99
|
-
* via decodeSymlinkMetadataValue, but new uploads use
|
|
100
|
-
* SYMLINK_MARKER_META_VALUE — the target lives in the body now.
|
|
101
|
-
*/
|
|
102
|
-
export function encodeSymlinkMetadataValue(target: string): string {
|
|
103
|
-
return Buffer.from(target, "utf-8").toString("base64");
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
/**
|
|
107
|
-
* Decode a target from the S3 metadata header value. Used as a
|
|
108
|
-
* legacy fallback when the body doesn't carry SYMLINK_BODY_PREFIX
|
|
109
|
-
* (i.e. an in-flight upload from earlier in this PR before the
|
|
110
|
-
* marker-only metadata convention). Round-trip-validates: if the
|
|
111
|
-
* value isn't valid base64 of UTF-8, returns the raw string.
|
|
112
|
-
*/
|
|
113
|
-
export function decodeSymlinkMetadataValue(value: string): string {
|
|
114
|
-
try {
|
|
115
|
-
const decoded = Buffer.from(value, "base64").toString("utf-8");
|
|
116
|
-
if (Buffer.from(decoded, "utf-8").toString("base64") === value) {
|
|
117
|
-
return decoded;
|
|
118
|
-
}
|
|
119
|
-
} catch {
|
|
120
|
-
// fall through
|
|
121
|
-
}
|
|
122
|
-
return value;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
/**
|
|
126
|
-
* Magic prefix prepended to symlink-record bodies on the wire. Two
|
|
127
|
-
* properties this gives us:
|
|
128
|
-
*
|
|
129
|
-
* 1. ETag distinguishability. S3 ETag = MD5(body). Without a prefix,
|
|
130
|
-
* a symlink whose target string equals some regular file's exact
|
|
131
|
-
* contents would produce the same ETag, and the LIST-based pull
|
|
132
|
-
* planner (which can't see per-object metadata) would classify a
|
|
133
|
-
* symlink ↔ regular-file transition as "no change" and never
|
|
134
|
-
* replace the local representation. The prefix makes those two
|
|
135
|
-
* shapes ETag-distinguishable for the realistic case (collision
|
|
136
|
-
* now requires a regular file whose contents *start* with this
|
|
137
|
-
* prefix, which is implausible for any non-malicious source).
|
|
138
|
-
*
|
|
139
|
-
* 2. Fallback discriminator. If user-metadata is ever lost (S3
|
|
140
|
-
* cross-region replication of object data only, manual S3 console
|
|
141
|
-
* copy that drops Metadata), the body prefix lets a downloader
|
|
142
|
-
* recover the symlink record without needing the metadata header.
|
|
143
|
-
* We don't currently rely on this fallback — the metadata header
|
|
144
|
-
* is still the primary discriminator on the read path — but the
|
|
145
|
-
* prefix keeps the option open and avoids painting us into a
|
|
146
|
-
* "metadata is the only signal" corner.
|
|
147
|
-
*
|
|
148
|
-
* Format: `hq-symlink:` + target string (UTF-8 bytes). No trailing
|
|
149
|
-
* newline. The colon separates the marker from the target so a future
|
|
150
|
-
* extension can encode additional fields if needed.
|
|
151
|
-
*/
|
|
152
|
-
export const SYMLINK_BODY_PREFIX = "hq-symlink:";
|
|
153
|
-
const SYMLINK_BODY_PREFIX_BYTES = Buffer.from(SYMLINK_BODY_PREFIX, "utf-8");
|
|
154
|
-
|
|
155
|
-
/**
|
|
156
|
-
* S3 user-metadata key carrying the source-side file mode (permission bits
|
|
157
|
-
* only — \`mode & 0o777\`) as an octal string ("755", "640", etc.). On
|
|
158
|
-
* download, downloadFile parses this with \`parseInt(value, 8)\` and chmods
|
|
159
|
-
* the file to the exact source mode after the byte write.
|
|
160
|
-
*
|
|
161
|
-
* Bug #5 in the 5.33.0 deep-test was originally reported as "exec bit lost
|
|
162
|
-
* on sync" but the verification report broadened it: ALL modes (0600 / 0640
|
|
163
|
-
* / 0700 / 0750 / 0755) collapsed to the receiver's umask default (0644)
|
|
164
|
-
* because no mode signal crossed the wire at all. Stamping the mode in
|
|
165
|
-
* metadata is the smallest schema change that preserves the full
|
|
166
|
-
* permission bitfield without a per-host umask negotiation.
|
|
167
|
-
*
|
|
168
|
-
* Symlinks: skipped at upload time (symlink mode is OS-controlled
|
|
169
|
-
* lrwxrwxrwx) and skipped on download (\`fs.chmodSync\` follows symlinks
|
|
170
|
-
* and would mutate the target's mode instead).
|
|
171
|
-
*
|
|
172
|
-
* Back-compat: legacy uploads have no \`hq-mode\` header — the receiver
|
|
173
|
-
* leaves the umask default in place, matching pre-fix behavior.
|
|
174
|
-
*/
|
|
175
|
-
export const FILE_MODE_META_KEY = "hq-mode";
|
|
176
|
-
|
|
177
|
-
/**
|
|
178
|
-
* S3 user-metadata key carrying the source-side file modification time
|
|
179
|
-
* (mtimeMs) as an integer-millisecond epoch string ("1700000000000"). On
|
|
180
|
-
* download, downloadFile parses this with a strict-numeric regex BEFORE
|
|
181
|
-
* parseInt, then applies it via `fs.utimesSync(localPath, mtimeDate,
|
|
182
|
-
* mtimeDate)` after the byte write.
|
|
183
|
-
*
|
|
184
|
-
* 5.37.0 symmetric to the 5.34.0 Bug #5 mode preservation: a file's
|
|
185
|
-
* modification time should follow it across machines instead of resetting
|
|
186
|
-
* to "the time of sync." Without this metadata, every receiver's mtime is
|
|
187
|
-
* wall-clock-now at write-time — making "newer than" comparisons,
|
|
188
|
-
* mtime-keyed caches, and reproducible builds break across sync.
|
|
189
|
-
*
|
|
190
|
-
* Symlinks: skipped at upload time (symlink mtime is OS-controlled and
|
|
191
|
-
* `lstat` on a symlink already returns the symlink's own times — but we
|
|
192
|
-
* don't stamp them because the symlink record wire body is `hq-symlink:`
|
|
193
|
-
* + target string, not real file content, so its mtime isn't user-
|
|
194
|
-
* meaningful) and skipped on download (`fs.utimesSync` follows symlinks
|
|
195
|
-
* and would mutate the target's mtime instead; `lutimesSync` is not in
|
|
196
|
-
* stable Node).
|
|
197
|
-
*
|
|
198
|
-
* Composition with 5.36.0 lstat fast-path: the journal stamp is captured
|
|
199
|
-
* AFTER utimesSync runs (the share/sync call sites lstat AFTER
|
|
200
|
-
* downloadFile returns), so the journal's mtimeMs matches the post-utimes
|
|
201
|
-
* lstat. The next sync's fast-path correctly skips re-hashing.
|
|
202
|
-
*
|
|
203
|
-
* Clock skew: a peer with a wrong clock pushes file with mtimeMs=<wrong>;
|
|
204
|
-
* receivers apply <wrong>. This is the same trade git's "file from the
|
|
205
|
-
* future" warning makes — silent in our case, deliberately. Clock skew
|
|
206
|
-
* is the user's problem, not the sync engine's.
|
|
207
|
-
*
|
|
208
|
-
* Back-compat: legacy uploads (pre-5.37.0) have no `hq-mtime` header —
|
|
209
|
-
* the receiver leaves the on-disk mtime at write-time, matching pre-
|
|
210
|
-
* 5.37.0 behavior. Forward-compat: pre-5.37.0 pullers ignore `hq-mtime`
|
|
211
|
-
* and keep their current "mtime = write-time" behavior. Both work; only
|
|
212
|
-
* the receiver upgrade unlocks the feature.
|
|
213
|
-
*/
|
|
214
|
-
export const FILE_MTIME_META_KEY = "hq-mtime";
|
|
215
|
-
|
|
216
|
-
/**
|
|
217
|
-
* S3 user-metadata key carrying the source-side file birthtime (birthtimeMs)
|
|
218
|
-
* as an integer-millisecond epoch string. Stamped on upload ONLY when
|
|
219
|
-
* `birthtimeMs > 0 && birthtimeMs !== mtimeMs` — many filesystems (Linux
|
|
220
|
-
* ext4 historically, tmpfs, some FUSE mounts) return 0 (unsupported) or
|
|
221
|
-
* the same value as mtime (no separate creation time tracking). The
|
|
222
|
-
* filter keeps the metadata header free of noise on those platforms.
|
|
223
|
-
*
|
|
224
|
-
* Pull: NO-OP for now. Node has no API to set birthtime on POSIX as of
|
|
225
|
-
* v24 (no `lbirthtime`, no `birthtimeSync`). The push side stamps it
|
|
226
|
-
* anyway so a future receiver upgrade — once Node lands the API — can
|
|
227
|
-
* apply it without a server-side data migration.
|
|
228
|
-
*
|
|
229
|
-
* Symlinks: skipped on both sides for the same reasons as `hq-mtime`.
|
|
230
|
-
*/
|
|
231
|
-
export const FILE_BTIME_META_KEY = "hq-btime";
|
|
232
|
-
|
|
233
|
-
/**
|
|
234
|
-
* Encode/decode the symlink wire body. Kept as exported helpers so the
|
|
235
|
-
* format is centrally defined and tests can probe both sides without
|
|
236
|
-
* duplicating the prefix string.
|
|
237
|
-
*/
|
|
238
|
-
export function encodeSymlinkBody(target: string): Buffer {
|
|
239
|
-
return Buffer.from(SYMLINK_BODY_PREFIX + target, "utf-8");
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
/**
|
|
243
|
-
* Staged-download (`.hq-tmp-*`) and replacement-backup (`.hq-backup-*`)
|
|
244
|
-
* sentinels are cleaned by try/finally in-process, but a hard kill
|
|
245
|
-
* mid-transfer (agent job `timeout -k`, OOM, systemd stop) never reaches the
|
|
246
|
-
* finally and nothing else removes them. On a fleet box that leaked 866M of
|
|
247
|
-
* orphans in two days and filled /tmp (2026-07-20 incident). Sweep stale
|
|
248
|
-
* sentinels in the destination dir before staging a new one there.
|
|
249
|
-
*
|
|
250
|
-
* The age threshold far exceeds any live transfer, so a concurrent download's
|
|
251
|
-
* fresh sentinel is never touched. Only non-directories matching the sentinel
|
|
252
|
-
* pattern are candidates. Per-dir memoization bounds the readdir cost to once
|
|
253
|
-
* per directory per process run.
|
|
254
|
-
*/
|
|
255
|
-
const STAGED_SENTINEL_PATTERN = /^\.hq-(tmp|backup)-/;
|
|
256
|
-
const STAGED_SWEEP_MAX_AGE_MS = 60 * 60 * 1000;
|
|
257
|
-
const sweptStagedDirs = new Set<string>();
|
|
258
|
-
|
|
259
|
-
export function sweepStaleStagedFiles(
|
|
260
|
-
dir: string,
|
|
261
|
-
maxAgeMs: number = STAGED_SWEEP_MAX_AGE_MS,
|
|
262
|
-
nowMs: number = Date.now(),
|
|
263
|
-
): string[] {
|
|
264
|
-
let entries: string[];
|
|
265
|
-
try {
|
|
266
|
-
entries = fs.readdirSync(dir);
|
|
267
|
-
} catch (err) {
|
|
268
|
-
// Destination dir may not exist yet (created later by downloadFile) —
|
|
269
|
-
// nothing to sweep. Anything else is unexpected but must not fail the
|
|
270
|
-
// download that triggered the sweep.
|
|
271
|
-
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
272
|
-
console.warn(`[hq-sync] staged-file sweep: cannot read ${dir}: ${err}`);
|
|
273
|
-
}
|
|
274
|
-
return [];
|
|
275
|
-
}
|
|
276
|
-
const removed: string[] = [];
|
|
277
|
-
for (const name of entries) {
|
|
278
|
-
if (!STAGED_SENTINEL_PATTERN.test(name)) continue;
|
|
279
|
-
const full = path.join(dir, name);
|
|
280
|
-
try {
|
|
281
|
-
const stat = fs.lstatSync(full);
|
|
282
|
-
if (stat.isDirectory()) continue;
|
|
283
|
-
if (nowMs - stat.mtimeMs < maxAgeMs) continue;
|
|
284
|
-
fs.unlinkSync(full);
|
|
285
|
-
removed.push(full);
|
|
286
|
-
} catch (err) {
|
|
287
|
-
// A concurrent process may have removed it first (ENOENT: fine), or the
|
|
288
|
-
// file is unremovable (EPERM/EROFS) — either way the download must
|
|
289
|
-
// proceed; surface non-ENOENT so the leak stays visible.
|
|
290
|
-
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
291
|
-
console.warn(`[hq-sync] staged-file sweep: cannot remove ${full}: ${err}`);
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
if (removed.length > 0) {
|
|
296
|
-
console.warn(
|
|
297
|
-
`[hq-sync] removed ${removed.length} stale staged file(s) in ${dir}`,
|
|
298
|
-
);
|
|
299
|
-
}
|
|
300
|
-
return removed;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
function sweepStaleStagedFilesOnce(dir: string): void {
|
|
304
|
-
if (sweptStagedDirs.has(dir)) return;
|
|
305
|
-
sweptStagedDirs.add(dir);
|
|
306
|
-
sweepStaleStagedFiles(dir);
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
function downloadTempPath(localPath: string): string {
|
|
310
|
-
const dir = path.dirname(localPath);
|
|
311
|
-
sweepStaleStagedFilesOnce(dir);
|
|
312
|
-
const random = crypto.randomBytes(10).toString("hex");
|
|
313
|
-
return path.join(dir, `.hq-tmp-${random}`);
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
function removeTempPath(tempPath: string): void {
|
|
317
|
-
try {
|
|
318
|
-
fs.unlinkSync(tempPath);
|
|
319
|
-
} catch {
|
|
320
|
-
// A win32 directory-type symlink is a directory reparse point and
|
|
321
|
-
// refuses unlink (EPERM); rmdir is the correct removal call for it.
|
|
322
|
-
try {
|
|
323
|
-
fs.rmdirSync(tempPath);
|
|
324
|
-
} catch {
|
|
325
|
-
// Best-effort cleanup; do not mask the transfer/materialization error.
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
export interface CreateStagedSymlinkOps {
|
|
331
|
-
platform: NodeJS.Platform;
|
|
332
|
-
symlink(target: string, linkPath: string, type?: fs.symlink.Type): void;
|
|
333
|
-
statIsDirectory(absPath: string): boolean | undefined;
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
const DEFAULT_CREATE_STAGED_SYMLINK_OPS: CreateStagedSymlinkOps = {
|
|
337
|
-
platform: process.platform,
|
|
338
|
-
symlink: (target, linkPath, type) =>
|
|
339
|
-
fs.symlinkSync(target, linkPath, type),
|
|
340
|
-
statIsDirectory: (absPath) => {
|
|
341
|
-
try {
|
|
342
|
-
return fs.statSync(absPath).isDirectory();
|
|
343
|
-
} catch {
|
|
344
|
-
return undefined;
|
|
345
|
-
}
|
|
346
|
-
},
|
|
347
|
-
};
|
|
348
|
-
|
|
349
|
-
/**
|
|
350
|
-
* Create a fully-staged symlink without requiring Windows Developer Mode.
|
|
351
|
-
* NTFS symlinks fail with EPERM without SeCreateSymbolicLinkPrivilege, while
|
|
352
|
-
* directory junctions need no privilege but require an absolute target. HQ
|
|
353
|
-
* vault links are directory overlays, so missing targets default to junctions.
|
|
354
|
-
* See Sentry HQ-DESKTOP-3M.
|
|
355
|
-
*/
|
|
356
|
-
export function createStagedSymlink(
|
|
357
|
-
target: string,
|
|
358
|
-
linkPath: string,
|
|
359
|
-
ops: CreateStagedSymlinkOps = DEFAULT_CREATE_STAGED_SYMLINK_OPS,
|
|
360
|
-
): void {
|
|
361
|
-
if (ops.platform !== "win32") {
|
|
362
|
-
ops.symlink(target, linkPath);
|
|
363
|
-
return;
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
const absTarget = path.isAbsolute(target)
|
|
367
|
-
? target
|
|
368
|
-
: path.resolve(path.dirname(linkPath), target);
|
|
369
|
-
if (ops.statIsDirectory(absTarget) === false) {
|
|
370
|
-
ops.symlink(target, linkPath, "file");
|
|
371
|
-
return;
|
|
372
|
-
}
|
|
373
|
-
|
|
374
|
-
ops.symlink(absTarget, linkPath, "junction");
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
/**
|
|
378
|
-
* downloadFile's `options.win32` seam is the authority for symlink-flavor
|
|
379
|
-
* decisions (it is what makes Windows semantics testable on POSIX CI), so the
|
|
380
|
-
* staged-link helper must see the SAME platform the idempotence fast path
|
|
381
|
-
* reasoned about rather than the raw `process.platform`. createStagedSymlink
|
|
382
|
-
* only ever asks "is this win32?", so "linux" is a faithful stand-in for
|
|
383
|
-
* "not Windows" on the false branch.
|
|
384
|
-
*/
|
|
385
|
-
function stagedSymlinkOpsFor(win32: boolean): CreateStagedSymlinkOps {
|
|
386
|
-
return {
|
|
387
|
-
...DEFAULT_CREATE_STAGED_SYMLINK_OPS,
|
|
388
|
-
platform: win32 ? "win32" : "linux",
|
|
389
|
-
};
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
/**
|
|
393
|
-
* Would createStagedSymlink() materialize `target` as a DIRECTORY-flavored
|
|
394
|
-
* link (a junction) rather than a file symlink? Mirrors that helper's rule
|
|
395
|
-
* exactly — only a target confirmed to be a non-directory earns a file
|
|
396
|
-
* symlink; directories and unresolvable targets become junctions, which need
|
|
397
|
-
* no Windows privilege. Keeping the two in lockstep is what lets the
|
|
398
|
-
* idempotence fast path below decide whether an existing link is already the
|
|
399
|
-
* link we would mint today.
|
|
400
|
-
*/
|
|
401
|
-
function win32StagedLinkIsDirectoryFlavored(
|
|
402
|
-
target: string,
|
|
403
|
-
linkPath: string,
|
|
404
|
-
ops: CreateStagedSymlinkOps = DEFAULT_CREATE_STAGED_SYMLINK_OPS,
|
|
405
|
-
): boolean {
|
|
406
|
-
const absTarget = path.isAbsolute(target)
|
|
407
|
-
? target
|
|
408
|
-
: path.resolve(path.dirname(linkPath), target);
|
|
409
|
-
return ops.statIsDirectory(absTarget) !== false;
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
/**
|
|
413
|
-
* Compare an on-disk readlink() result against an incoming wire target.
|
|
414
|
-
*
|
|
415
|
-
* POSIX compares exactly: we write the wire target verbatim, so readlink()
|
|
416
|
-
* returns it byte-for-byte, and a backslash is an ordinary filename character
|
|
417
|
-
* there. On win32 neither property holds — createStagedSymlink() materializes
|
|
418
|
-
* directory overlays as JUNCTIONS, which require an ABSOLUTE substitute path,
|
|
419
|
-
* and Node rewrites '/' → '\\' at symlink() time. A raw string compare would
|
|
420
|
-
* therefore never match on Windows and the fast path would never fire for the
|
|
421
|
-
* .claude/skills link farm (the exact perpetual-churn bug this guards). So
|
|
422
|
-
* win32 equality resolves BOTH sides against the link's parent directory and
|
|
423
|
-
* compares separator-insensitively, which normalizes the relative wire target
|
|
424
|
-
* and the absolute junction target onto the same footing.
|
|
425
|
-
*/
|
|
426
|
-
function symlinkTargetsEqual(
|
|
427
|
-
existing: string,
|
|
428
|
-
incoming: string,
|
|
429
|
-
linkPath: string,
|
|
430
|
-
win32: boolean,
|
|
431
|
-
): boolean {
|
|
432
|
-
if (!win32) return existing === incoming;
|
|
433
|
-
const normalize = (value: string): string =>
|
|
434
|
-
path
|
|
435
|
-
.resolve(path.dirname(linkPath), value.replace(/\\/g, "/"))
|
|
436
|
-
.replace(/\\/g, "/");
|
|
437
|
-
return normalize(existing) === normalize(incoming);
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
export interface ReplaceStagedPathOps {
|
|
441
|
-
lstat(path: string): fs.Stats;
|
|
442
|
-
rename(from: string, to: string): void;
|
|
443
|
-
remove(path: string): void;
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
const DEFAULT_REPLACE_STAGED_PATH_OPS: ReplaceStagedPathOps = {
|
|
447
|
-
lstat: (p) => fs.lstatSync(p),
|
|
448
|
-
rename: (from, to) => fs.renameSync(from, to),
|
|
449
|
-
remove: (p) => {
|
|
450
|
-
try {
|
|
451
|
-
fs.unlinkSync(p);
|
|
452
|
-
} catch (err) {
|
|
453
|
-
// A win32 directory-type symlink or junction is a directory reparse
|
|
454
|
-
// point; some runtimes refuse unlink (EPERM/EISDIR) and need rmdir —
|
|
455
|
-
// same fallback as removeTempPath. Anything else stays loud so
|
|
456
|
-
// replaceStagedPath can roll back.
|
|
457
|
-
const code = (err as NodeJS.ErrnoException).code;
|
|
458
|
-
if (code !== "EPERM" && code !== "EISDIR") throw err;
|
|
459
|
-
fs.rmdirSync(p);
|
|
460
|
-
}
|
|
461
|
-
},
|
|
462
|
-
};
|
|
463
|
-
|
|
464
|
-
function replacementBackupPath(localPath: string): string {
|
|
465
|
-
const dir = path.dirname(localPath);
|
|
466
|
-
const random = crypto.randomBytes(10).toString("hex");
|
|
467
|
-
return path.join(dir, `.hq-backup-${random}`);
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
/**
|
|
471
|
-
* Install a fully-staged download over its destination.
|
|
472
|
-
*
|
|
473
|
-
* A direct rename is atomic and remains the fast path for absent destinations
|
|
474
|
-
* and ordinary files. Windows cannot rename over an existing directory link,
|
|
475
|
-
* though, because MoveFileEx treats that link as an existing directory. For a
|
|
476
|
-
* symlink/junction destination, move the old link to a sibling backup first,
|
|
477
|
-
* install the staged entry, then remove the backup. Any failure after the
|
|
478
|
-
* backup move restores the exact prior link before rethrowing.
|
|
479
|
-
*
|
|
480
|
-
* The tiny operations seam keeps the Windows failure/rollback contract
|
|
481
|
-
* deterministic in ESM tests without spying on non-configurable fs exports.
|
|
482
|
-
*/
|
|
483
|
-
export function replaceStagedPath(
|
|
484
|
-
stagedPath: string,
|
|
485
|
-
localPath: string,
|
|
486
|
-
ops: ReplaceStagedPathOps = DEFAULT_REPLACE_STAGED_PATH_OPS,
|
|
487
|
-
): void {
|
|
488
|
-
let existing: fs.Stats;
|
|
489
|
-
try {
|
|
490
|
-
existing = ops.lstat(localPath);
|
|
491
|
-
} catch (err) {
|
|
492
|
-
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
|
493
|
-
ops.rename(stagedPath, localPath);
|
|
494
|
-
return;
|
|
495
|
-
}
|
|
496
|
-
throw err;
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
if (!existing.isSymbolicLink()) {
|
|
500
|
-
ops.rename(stagedPath, localPath);
|
|
501
|
-
return;
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
const backupPath = replacementBackupPath(localPath);
|
|
505
|
-
ops.rename(localPath, backupPath);
|
|
506
|
-
let stagedInstalled = false;
|
|
507
|
-
try {
|
|
508
|
-
ops.rename(stagedPath, localPath);
|
|
509
|
-
stagedInstalled = true;
|
|
510
|
-
ops.remove(backupPath);
|
|
511
|
-
} catch (primaryError) {
|
|
512
|
-
const rollbackErrors: unknown[] = [];
|
|
513
|
-
if (stagedInstalled) {
|
|
514
|
-
try {
|
|
515
|
-
ops.rename(localPath, stagedPath);
|
|
516
|
-
} catch (rollbackError) {
|
|
517
|
-
rollbackErrors.push(rollbackError);
|
|
518
|
-
}
|
|
519
|
-
}
|
|
520
|
-
try {
|
|
521
|
-
ops.rename(backupPath, localPath);
|
|
522
|
-
} catch (rollbackError) {
|
|
523
|
-
rollbackErrors.push(rollbackError);
|
|
524
|
-
}
|
|
525
|
-
if (rollbackErrors.length > 0) {
|
|
526
|
-
throw new AggregateError(
|
|
527
|
-
[primaryError, ...rollbackErrors],
|
|
528
|
-
"Staged path replacement failed and the previous directory link could not be fully restored",
|
|
529
|
-
);
|
|
530
|
-
}
|
|
531
|
-
throw primaryError;
|
|
532
|
-
}
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
async function getObjectStream(
|
|
536
|
-
io: ObjectIO,
|
|
537
|
-
key: string,
|
|
538
|
-
): Promise<GetObjectStreamResult> {
|
|
539
|
-
if (io.getObjectStream) return io.getObjectStream(key);
|
|
540
|
-
const res = await io.getObject(key);
|
|
541
|
-
return {
|
|
542
|
-
body: (async function* () {
|
|
543
|
-
yield res.body;
|
|
544
|
-
})(),
|
|
545
|
-
metadata: res.metadata,
|
|
546
|
-
};
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
function bufferFromChunk(chunk: Uint8Array): Buffer {
|
|
550
|
-
if (Buffer.isBuffer(chunk)) return chunk;
|
|
551
|
-
return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
function leadingBytes(chunks: Buffer[], totalLength: number, length: number): Buffer {
|
|
555
|
-
const out = Buffer.alloc(Math.min(totalLength, length));
|
|
556
|
-
let offset = 0;
|
|
557
|
-
for (const chunk of chunks) {
|
|
558
|
-
if (offset >= out.length) break;
|
|
559
|
-
const take = Math.min(chunk.length, out.length - offset);
|
|
560
|
-
chunk.copy(out, offset, 0, take);
|
|
561
|
-
offset += take;
|
|
562
|
-
}
|
|
563
|
-
return out;
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
async function collectRemainingChunks(
|
|
567
|
-
initialChunks: Buffer[],
|
|
568
|
-
iterator: AsyncIterator<Uint8Array>,
|
|
569
|
-
): Promise<Buffer> {
|
|
570
|
-
const chunks = [...initialChunks];
|
|
571
|
-
let totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
|
572
|
-
while (true) {
|
|
573
|
-
const next = await iterator.next();
|
|
574
|
-
if (next.done) break;
|
|
575
|
-
const chunk = bufferFromChunk(next.value);
|
|
576
|
-
if (chunk.length === 0) continue;
|
|
577
|
-
chunks.push(chunk);
|
|
578
|
-
totalLength += chunk.length;
|
|
579
|
-
}
|
|
580
|
-
return Buffer.concat(chunks, totalLength);
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
async function* regularDownloadChunks(
|
|
584
|
-
initialChunks: Buffer[],
|
|
585
|
-
iterator: AsyncIterator<Uint8Array>,
|
|
586
|
-
): AsyncIterable<Buffer> {
|
|
587
|
-
for (const chunk of initialChunks) {
|
|
588
|
-
if (chunk.length > 0) yield chunk;
|
|
589
|
-
}
|
|
590
|
-
while (true) {
|
|
591
|
-
const next = await iterator.next();
|
|
592
|
-
if (next.done) break;
|
|
593
|
-
const chunk = bufferFromChunk(next.value);
|
|
594
|
-
if (chunk.length > 0) yield chunk;
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
class HashingTransform extends Transform {
|
|
599
|
-
private readonly hash = crypto.createHash("sha256");
|
|
600
|
-
size = 0;
|
|
601
|
-
|
|
602
|
-
_transform(
|
|
603
|
-
chunk: Buffer,
|
|
604
|
-
_encoding: BufferEncoding,
|
|
605
|
-
callback: (error?: Error | null, data?: Buffer) => void,
|
|
606
|
-
): void {
|
|
607
|
-
this.hash.update(chunk);
|
|
608
|
-
this.size += chunk.length;
|
|
609
|
-
callback(null, chunk);
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
digest(): string {
|
|
613
|
-
return this.hash.digest("hex");
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
|
|
617
|
-
async function streamRegularFileToTemp(
|
|
618
|
-
tempPath: string,
|
|
619
|
-
initialChunks: Buffer[],
|
|
620
|
-
iterator: AsyncIterator<Uint8Array>,
|
|
621
|
-
): Promise<{ hash: string; size: number }> {
|
|
622
|
-
const hashing = new HashingTransform();
|
|
623
|
-
await pipeline(
|
|
624
|
-
Readable.from(regularDownloadChunks(initialChunks, iterator)),
|
|
625
|
-
hashing,
|
|
626
|
-
fs.createWriteStream(tempPath, { flags: "wx" }),
|
|
627
|
-
);
|
|
628
|
-
return { hash: hashing.digest(), size: hashing.size };
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
/**
|
|
632
|
-
* Batch pre-mint transport URLs for `keys` under `op` so the subsequent
|
|
633
|
-
* per-file transfer calls (downloadFile/headRemoteFile/…) reuse them instead
|
|
634
|
-
* of presigning one key at a time. On the presigned-URL transport this turns
|
|
635
|
-
* an N-file leg from N presign requests into ceil(N/100) — the difference
|
|
636
|
-
* between completing a bulk pull and 429ing past the 100-req/hr limit. No-op
|
|
637
|
-
* on the S3 SDK transport (which has no presign step) and harmless if called
|
|
638
|
-
* with an empty list. Best-effort: a prime failure never propagates — the
|
|
639
|
-
* per-file path falls back to a single presign.
|
|
640
|
-
*
|
|
641
|
-
* Call it once, right before a transfer loop, with the full key set the loop
|
|
642
|
-
* will touch. The presigned transport memoizes one IO instance per company for
|
|
643
|
-
* the run, so the warmed cache is the same one the loop drains.
|
|
644
|
-
*/
|
|
645
|
-
export async function primeObjectTransport(
|
|
646
|
-
ctx: EntityContext,
|
|
647
|
-
op: "get" | "put" | "delete",
|
|
648
|
-
keys: string[],
|
|
649
|
-
): Promise<void> {
|
|
650
|
-
if (keys.length === 0) return;
|
|
651
|
-
const io = resolveObjectIO(ctx);
|
|
652
|
-
if (!io.prime) return;
|
|
653
|
-
await io.prime(
|
|
654
|
-
op,
|
|
655
|
-
keys.map((key) => ({ key })),
|
|
656
|
-
);
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
/**
|
|
660
|
-
* Source-side mode + mtime (+ btime when distinct) metadata for a regular
|
|
661
|
-
* file, from a single lstat. Symlinks carry none (OS-controlled mode; a link's
|
|
662
|
-
* mtime isn't user-meaningful — the wire body is the target string, not file
|
|
663
|
-
* content). Shared by uploadFile and the primeUploads pre-pass so the PUT
|
|
664
|
-
* metadata they produce is byte-identical. See the FILE_*_META_KEY docs for the
|
|
665
|
-
* per-field rationale.
|
|
666
|
-
*/
|
|
667
|
-
function buildModeTimeMetadata(lstat: fs.Stats): Record<string, string> {
|
|
668
|
-
const meta: Record<string, string> = {};
|
|
669
|
-
if (lstat.isSymbolicLink()) return meta;
|
|
670
|
-
meta[FILE_MODE_META_KEY] = (lstat.mode & 0o777).toString(8);
|
|
671
|
-
const mtimeFloor = Math.floor(lstat.mtimeMs);
|
|
672
|
-
if (Number.isFinite(lstat.mtimeMs)) meta[FILE_MTIME_META_KEY] = String(mtimeFloor);
|
|
673
|
-
const btimeFloor = Math.floor(lstat.birthtimeMs);
|
|
674
|
-
if (
|
|
675
|
-
Number.isFinite(lstat.birthtimeMs) &&
|
|
676
|
-
btimeFloor > 0 &&
|
|
677
|
-
btimeFloor !== mtimeFloor
|
|
678
|
-
) {
|
|
679
|
-
meta[FILE_BTIME_META_KEY] = String(btimeFloor);
|
|
680
|
-
}
|
|
681
|
-
return meta;
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
/**
|
|
685
|
-
* Resolve the created-at to stamp: the existing object's value (preserved
|
|
686
|
-
* across re-uploads so the hq-console NEW-pill window doesn't reset) or now for
|
|
687
|
-
* a first upload. HEAD failure / no author → now. Shared by upload* and
|
|
688
|
-
* primeUploads so both agree on the value signed into the PUT.
|
|
689
|
-
*/
|
|
690
|
-
async function resolveCreatedAt(
|
|
691
|
-
io: ObjectIO,
|
|
692
|
-
key: string,
|
|
693
|
-
author?: UploadAuthor,
|
|
694
|
-
): Promise<string> {
|
|
695
|
-
let createdAt = new Date().toISOString();
|
|
696
|
-
if (author) {
|
|
697
|
-
try {
|
|
698
|
-
const head = await io.headObject(key);
|
|
699
|
-
const existing = head?.metadata?.["created-at"];
|
|
700
|
-
if (typeof existing === "string" && existing.length > 0) {
|
|
701
|
-
createdAt = existing;
|
|
702
|
-
}
|
|
703
|
-
} catch {
|
|
704
|
-
// Object doesn't exist yet, or HEAD failed — keep now (first upload).
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
return createdAt;
|
|
708
|
-
}
|
|
709
|
-
|
|
710
|
-
/**
|
|
711
|
-
* One upload's identity for {@link primeUploads}: the vault key, the local
|
|
712
|
-
* path (to lstat for mode/mtime), whether it's a symlink, and the author.
|
|
713
|
-
*/
|
|
714
|
-
export interface UploadPrimeItem {
|
|
715
|
-
key: string;
|
|
716
|
-
localPath: string;
|
|
717
|
-
isSymlink: boolean;
|
|
718
|
-
author?: UploadAuthor;
|
|
719
|
-
}
|
|
720
|
-
|
|
721
|
-
/**
|
|
722
|
-
* Batch pre-mint PUT URLs (+ the created-at HEADs they depend on) for a set of
|
|
723
|
-
* uploads, signing the SAME metadata uploadFile/uploadSymlink would compute so
|
|
724
|
-
* the transfer loop can replay the cached headers. Turns an N-file push from
|
|
725
|
-
* ~N presign calls (1 per PUT, sometimes 2-3 with HEADs) into ceil(N/1000) GET
|
|
726
|
-
* + ceil(N/1000) PUT — the difference between completing a bulk push and 429ing
|
|
727
|
-
* past the 100/hr limit. No-op on the S3 SDK transport; best-effort.
|
|
728
|
-
*
|
|
729
|
-
* The per-item created-at HEADs run over the GET cache primed first, so they
|
|
730
|
-
* cost S3 round-trips but NO extra presign calls (not counted against 100/hr).
|
|
731
|
-
*/
|
|
732
|
-
export async function primeUploads(
|
|
733
|
-
ctx: EntityContext,
|
|
734
|
-
items: UploadPrimeItem[],
|
|
735
|
-
): Promise<void> {
|
|
736
|
-
const io = resolveObjectIO(ctx);
|
|
737
|
-
if (!io.prime || items.length === 0) return;
|
|
738
|
-
|
|
739
|
-
// Prime GET first so each item's created-at HEAD reuses a cached URL.
|
|
740
|
-
// Canonicalize to POSIX here (one-canonical-form, matching the uploadFile /
|
|
741
|
-
// uploadSymlink boundary): a Windows-origin backslash key must cache under
|
|
742
|
-
// the SAME key uploadFile later looks up via hasPrimedPut(toPosixKey(...)),
|
|
743
|
-
// or the primed URL silently misses and the upload re-presigns. It also
|
|
744
|
-
// keeps the created-at HEAD pointed at the real (POSIX) object.
|
|
745
|
-
await io.prime(
|
|
746
|
-
"get",
|
|
747
|
-
items.map((i) => ({ key: toPosixKey(i.key) })),
|
|
748
|
-
);
|
|
749
|
-
|
|
750
|
-
// Build per-key PUT metadata with the SAME builders the upload path uses,
|
|
751
|
-
// bounded-concurrently (the HEADs are cheap cached-GET fetches).
|
|
752
|
-
const putKeys: Array<{
|
|
753
|
-
key: string;
|
|
754
|
-
contentType: string;
|
|
755
|
-
metadata: Record<string, string>;
|
|
756
|
-
}> = [];
|
|
757
|
-
const CONCURRENCY = 16;
|
|
758
|
-
let next = 0;
|
|
759
|
-
const worker = async (): Promise<void> => {
|
|
760
|
-
while (next < items.length) {
|
|
761
|
-
const it = items[next++];
|
|
762
|
-
// Same boundary guardrail as uploadFile/uploadSymlink: prime under the
|
|
763
|
-
// canonical POSIX key so the cached PUT URL is keyed identically to the
|
|
764
|
-
// hasPrimedPut/putObject lookup, and a backslash key can never be primed
|
|
765
|
-
// (let alone stored) as a non-POSIX vault key.
|
|
766
|
-
const key = toPosixKey(it.key);
|
|
767
|
-
const createdAt = await resolveCreatedAt(io, key, it.author);
|
|
768
|
-
if (it.isSymlink) {
|
|
769
|
-
putKeys.push({
|
|
770
|
-
key,
|
|
771
|
-
contentType: "application/octet-stream",
|
|
772
|
-
metadata: {
|
|
773
|
-
[SYMLINK_TARGET_META_KEY]: SYMLINK_MARKER_META_VALUE,
|
|
774
|
-
...(it.author ? buildAuthorMetadata(it.author, createdAt) : {}),
|
|
775
|
-
},
|
|
776
|
-
});
|
|
777
|
-
} else {
|
|
778
|
-
let modeTime: Record<string, string> = {};
|
|
779
|
-
try {
|
|
780
|
-
modeTime = buildModeTimeMetadata(fs.lstatSync(it.localPath));
|
|
781
|
-
} catch {
|
|
782
|
-
// raced rm / EPERM — leave stamps off (receiver umask default).
|
|
783
|
-
}
|
|
784
|
-
putKeys.push({
|
|
785
|
-
key,
|
|
786
|
-
contentType: getMimeType(key),
|
|
787
|
-
metadata: {
|
|
788
|
-
...(it.author ? buildAuthorMetadata(it.author, createdAt) : {}),
|
|
789
|
-
...modeTime,
|
|
790
|
-
},
|
|
791
|
-
});
|
|
792
|
-
}
|
|
793
|
-
}
|
|
794
|
-
};
|
|
795
|
-
await Promise.all(
|
|
796
|
-
Array.from({ length: Math.min(CONCURRENCY, items.length) }, worker),
|
|
797
|
-
);
|
|
798
|
-
|
|
799
|
-
await io.prime("put", putKeys);
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
/**
|
|
803
|
-
* Normalize an S3 object key to POSIX ("/") separators.
|
|
804
|
-
*
|
|
805
|
-
* S3 keys are always "/"-separated, and the vault listing reconstructs the
|
|
806
|
-
* folder tree by splitting keys on "/". A non-POSIX (Windows) sync client
|
|
807
|
-
* builds a key from `path.relative(...)`, whose separator is the native "\\";
|
|
808
|
-
* stored verbatim, a nested key like `knowledge\books-eoi.md` contains no "/"
|
|
809
|
-
* and the listing renders it flat at the vault root as one oddly-named file.
|
|
810
|
-
* Converting every "\\" to "/" makes the stored key POSIX regardless of the
|
|
811
|
-
* client OS.
|
|
812
|
-
*
|
|
813
|
-
* Mirrors the prefix guard in hq-pro files-acl (`FORBIDDEN_CHARS = /[?#\\[\]]/`,
|
|
814
|
-
* which already rejects "\\" on browse/grant prefixes). Uploads go direct to
|
|
815
|
-
* S3 via vended STS creds, so the server never re-validates the uploaded key —
|
|
816
|
-
* the upload primitives (uploadFile / uploadSymlink) are the only shared choke
|
|
817
|
-
* point, and they call this so a backslash key can never be stored again.
|
|
818
|
-
*/
|
|
819
|
-
export function toPosixKey(key: string): string {
|
|
820
|
-
return key.split("\\").join("/");
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
/**
|
|
824
|
-
* Client-side mirror of hq-pro's server validator
|
|
825
|
-
* (`vault-service/handlers/files-acl.ts validateObjectKey`). The direct-S3
|
|
826
|
-
* STS transport bypasses server key validation entirely — the upload
|
|
827
|
-
* primitives (uploadFile / uploadSymlink) are the only shared choke point —
|
|
828
|
-
* so we enforce the SAME rules here BEFORE the PUT. Incident 2026-07-11: a
|
|
829
|
-
* client with a stale doubled local tree (companies/{slug}/companies/{slug}/…)
|
|
830
|
-
* pushed literal `companies/…` keys into a company-scoped vault bucket via
|
|
831
|
-
* direct S3; agent boxes could LIST them but the presign download API
|
|
832
|
-
* rejected every one (INVALID_KEY_COMPANIES_SCOPED), erroring the whole
|
|
833
|
-
* company sync fleet-wide. A poisoned key must fail loudly at the writer,
|
|
834
|
-
* not at every reader.
|
|
835
|
-
*
|
|
836
|
-
* `scope` selects the companies/-prefix rule: company vault buckets are
|
|
837
|
-
* already company-scoped so a `companies/…` key is always wrong there;
|
|
838
|
-
* personal vaults legitimately carry `companies/{slug}/…` keys for local
|
|
839
|
-
* (non-cloud) companies. Callers derive scope from the EntityContext uid
|
|
840
|
-
* (`cmp_*` → "company", otherwise "personal").
|
|
841
|
-
*
|
|
842
|
-
* Errors carry the same `code` values the server throws so telemetry and
|
|
843
|
-
* operator messaging line up across client and server. Pull planning also
|
|
844
|
-
* uses the non-throwing classifier below to keep poison keys away from the
|
|
845
|
-
* presign transport.
|
|
846
|
-
*/
|
|
847
|
-
// Matching control chars is the whole point — block NUL/0x00–0x1F/0x7F in keys
|
|
848
|
-
// (they can smuggle past path checks or corrupt HTTP headers on the signed URL).
|
|
849
|
-
// eslint-disable-next-line no-control-regex
|
|
850
|
-
const KEY_CONTROL_CHARS = /[\x00-\x1F\x7F]/;
|
|
851
|
-
|
|
852
|
-
export type VaultKeyScope = "company" | "personal";
|
|
853
|
-
|
|
854
|
-
export interface VaultKeyValidationIssue {
|
|
855
|
-
code: string;
|
|
856
|
-
message: string;
|
|
857
|
-
}
|
|
858
|
-
|
|
859
|
-
/**
|
|
860
|
-
* Classify a vault key with the same rules as {@link validateVaultUploadKey}
|
|
861
|
-
* without throwing. Callers that enumerate remote objects can use this before
|
|
862
|
-
* presigning so permanently-invalid legacy keys remain benign skips.
|
|
863
|
-
*/
|
|
864
|
-
export function classifyVaultKey(
|
|
865
|
-
key: string,
|
|
866
|
-
scope: VaultKeyScope,
|
|
867
|
-
): VaultKeyValidationIssue | null {
|
|
868
|
-
if (key.length === 0) {
|
|
869
|
-
return {
|
|
870
|
-
code: "INVALID_KEY_EMPTY",
|
|
871
|
-
message: "Invalid key: must not be empty",
|
|
872
|
-
};
|
|
873
|
-
}
|
|
874
|
-
if (key.startsWith("/")) {
|
|
875
|
-
return {
|
|
876
|
-
code: "INVALID_KEY_LEADING_SLASH",
|
|
877
|
-
message: "Invalid key: leading '/' not allowed",
|
|
878
|
-
};
|
|
879
|
-
}
|
|
880
|
-
if (key.endsWith("/")) {
|
|
881
|
-
return {
|
|
882
|
-
code: "INVALID_KEY_TRAILING_SLASH",
|
|
883
|
-
message: "Invalid key: trailing '/' (folder marker, not an object)",
|
|
884
|
-
};
|
|
885
|
-
}
|
|
886
|
-
if (scope === "company" && key.startsWith("companies/")) {
|
|
887
|
-
return {
|
|
888
|
-
code: "INVALID_KEY_COMPANIES_SCOPED",
|
|
889
|
-
message:
|
|
890
|
-
"Invalid key: vault keys are bucket-relative and the vault is already " +
|
|
891
|
-
"company-scoped — do not prefix with 'companies/<slug>/'. This " +
|
|
892
|
-
"usually means a stale doubled local tree " +
|
|
893
|
-
"(companies/<slug>/companies/<slug>/…); remove the inner copy.",
|
|
894
|
-
};
|
|
895
|
-
}
|
|
896
|
-
if (KEY_CONTROL_CHARS.test(key)) {
|
|
897
|
-
return {
|
|
898
|
-
code: "INVALID_KEY_CONTROL_CHARS",
|
|
899
|
-
message: "Invalid key: contains control characters",
|
|
900
|
-
};
|
|
901
|
-
}
|
|
902
|
-
if (key.includes("\\")) {
|
|
903
|
-
return {
|
|
904
|
-
code: "INVALID_KEY_BACKSLASH",
|
|
905
|
-
message:
|
|
906
|
-
"Invalid key: backslash separators are not allowed — vault keys are POSIX ('/')",
|
|
907
|
-
};
|
|
908
|
-
}
|
|
909
|
-
if (key.includes("//")) {
|
|
910
|
-
return {
|
|
911
|
-
code: "INVALID_KEY_DOUBLE_SLASH",
|
|
912
|
-
message: "Invalid key: consecutive slashes not allowed",
|
|
913
|
-
};
|
|
914
|
-
}
|
|
915
|
-
for (const segment of key.split("/")) {
|
|
916
|
-
if (segment === "." || segment === "..") {
|
|
917
|
-
return {
|
|
918
|
-
code: "INVALID_KEY_DOT_COMPONENT",
|
|
919
|
-
message: "Invalid key: '.' and '..' path components are not allowed",
|
|
920
|
-
};
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
return null;
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
export function validateVaultUploadKey(
|
|
927
|
-
key: string,
|
|
928
|
-
scope: VaultKeyScope,
|
|
929
|
-
): void {
|
|
930
|
-
const issue = classifyVaultKey(key, scope);
|
|
931
|
-
if (issue !== null) {
|
|
932
|
-
throw Object.assign(new Error(issue.message), { code: issue.code });
|
|
933
|
-
}
|
|
934
|
-
}
|
|
935
|
-
|
|
936
|
-
/** Scope for {@link validateVaultUploadKey}, derived from the entity uid. */
|
|
937
|
-
function uploadScopeFor(ctx: EntityContext): VaultKeyScope {
|
|
938
|
-
return ctx.uid.startsWith("cmp_") ? "company" : "personal";
|
|
939
|
-
}
|
|
940
|
-
|
|
941
|
-
export async function uploadFile(
|
|
942
|
-
ctx: EntityContext,
|
|
943
|
-
localPath: string,
|
|
944
|
-
key: string,
|
|
945
|
-
author?: UploadAuthor,
|
|
946
|
-
precondition?: PutPrecondition,
|
|
947
|
-
): Promise<{ etag: string }> {
|
|
948
|
-
// Boundary guardrail: never store a non-POSIX key (see toPosixKey), and
|
|
949
|
-
// never store a key the server-side validator would reject (see
|
|
950
|
-
// validateVaultUploadKey — the direct-S3 STS transport bypasses server
|
|
951
|
-
// validation, so this is the last line of defense against key poisoning).
|
|
952
|
-
key = toPosixKey(key);
|
|
953
|
-
validateVaultUploadKey(key, uploadScopeFor(ctx));
|
|
954
|
-
const io = resolveObjectIO(ctx);
|
|
955
|
-
const body = fs.readFileSync(localPath);
|
|
956
|
-
|
|
957
|
-
// Fast path: a primeUploads() pre-pass already signed this file's metadata
|
|
958
|
-
// into a cached PUT URL. Skip the lstat-metadata + created-at HEAD and just
|
|
959
|
-
// send the body — putObject replays the cached headers (computed by the SAME
|
|
960
|
-
// builders below, so identical). hasPrimedPut only reports true with >60s of
|
|
961
|
-
// URL lifetime left, so the cache can't expire before the putObject below.
|
|
962
|
-
// Fenced PUTs bypass the fast path: a primed URL was signed WITHOUT the
|
|
963
|
-
// conditional header, so replaying it would silently drop the fence.
|
|
964
|
-
if (!precondition && io.hasPrimedPut?.(key)) {
|
|
965
|
-
const primed = await io.putObject({
|
|
966
|
-
key,
|
|
967
|
-
body,
|
|
968
|
-
contentType: getMimeType(key),
|
|
969
|
-
metadata: {},
|
|
970
|
-
});
|
|
971
|
-
return { etag: primed.etag };
|
|
972
|
-
}
|
|
973
|
-
|
|
974
|
-
// Source-side mode/mtime/btime (Bug #5 + 5.37.0) and the preserved
|
|
975
|
-
// created-at (so the NEW-pill window doesn't reset on re-upload). Both via
|
|
976
|
-
// the shared builders that primeUploads uses, so a primed PUT carries the
|
|
977
|
-
// identical metadata — see buildModeTimeMetadata / resolveCreatedAt.
|
|
978
|
-
let modeTime: Record<string, string> = {};
|
|
979
|
-
try {
|
|
980
|
-
modeTime = buildModeTimeMetadata(fs.lstatSync(localPath));
|
|
981
|
-
} catch {
|
|
982
|
-
// raced rm / EPERM — leave stamps off; receiver keeps its umask default.
|
|
983
|
-
}
|
|
984
|
-
const createdAt = await resolveCreatedAt(io, key, author);
|
|
985
|
-
const Metadata: Record<string, string> = {
|
|
986
|
-
...(author ? buildAuthorMetadata(author, createdAt) : {}),
|
|
987
|
-
...modeTime,
|
|
988
|
-
};
|
|
989
|
-
|
|
990
|
-
const response = await io.putObject({
|
|
991
|
-
key,
|
|
992
|
-
body,
|
|
993
|
-
contentType: getMimeType(key),
|
|
994
|
-
metadata: Metadata,
|
|
995
|
-
...(precondition ?? {}),
|
|
996
|
-
});
|
|
997
|
-
|
|
998
|
-
return { etag: response.etag };
|
|
999
|
-
}
|
|
1000
|
-
|
|
1001
|
-
/**
|
|
1002
|
-
* Upload a symlink as a zero-byte object whose user metadata carries the
|
|
1003
|
-
* link's target string. Mirrors uploadFile's signature so callers can pick
|
|
1004
|
-
* the right primitive once they've classified the entry as link vs file.
|
|
1005
|
-
*
|
|
1006
|
-
* The target string is stored verbatim — whatever fs.readlinkSync returned.
|
|
1007
|
-
* Relative targets transfer cleanly across machines; absolute targets are
|
|
1008
|
-
* preserved as-is and may be broken on a destination that doesn't share
|
|
1009
|
-
* the source's $HOME layout. Cross-machine portability of absolute targets
|
|
1010
|
-
* is out of scope for this primitive — the policy decision lives in the
|
|
1011
|
-
* caller (currently: upload anyway, never silently rewrite).
|
|
1012
|
-
*/
|
|
1013
|
-
export async function uploadSymlink(
|
|
1014
|
-
ctx: EntityContext,
|
|
1015
|
-
target: string,
|
|
1016
|
-
key: string,
|
|
1017
|
-
author?: UploadAuthor,
|
|
1018
|
-
precondition?: PutPrecondition,
|
|
1019
|
-
): Promise<{ etag: string }> {
|
|
1020
|
-
// Boundary guardrail: never store a non-POSIX key (see toPosixKey), and
|
|
1021
|
-
// never store a key the server-side validator would reject (see
|
|
1022
|
-
// validateVaultUploadKey).
|
|
1023
|
-
key = toPosixKey(key);
|
|
1024
|
-
validateVaultUploadKey(key, uploadScopeFor(ctx));
|
|
1025
|
-
const io = resolveObjectIO(ctx);
|
|
1026
|
-
const symlinkBody = encodeSymlinkBody(target);
|
|
1027
|
-
|
|
1028
|
-
// Fast path: primeUploads() already signed this symlink's metadata into a
|
|
1029
|
-
// cached PUT URL — send the body, replay the cached headers. Fenced PUTs
|
|
1030
|
-
// bypass it — the primed URL carries no conditional header (see uploadFile).
|
|
1031
|
-
if (!precondition && io.hasPrimedPut?.(key)) {
|
|
1032
|
-
const primed = await io.putObject({
|
|
1033
|
-
key,
|
|
1034
|
-
body: symlinkBody,
|
|
1035
|
-
contentType: "application/octet-stream",
|
|
1036
|
-
metadata: {},
|
|
1037
|
-
});
|
|
1038
|
-
return { etag: primed.etag };
|
|
1039
|
-
}
|
|
1040
|
-
|
|
1041
|
-
// Same created-at preservation as uploadFile (shared resolveCreatedAt) so the
|
|
1042
|
-
// NEW-pill window doesn't reset on re-upload, and so a primed PUT matches.
|
|
1043
|
-
const createdAt = await resolveCreatedAt(io, key, author);
|
|
1044
|
-
|
|
1045
|
-
const Metadata: Record<string, string> = {
|
|
1046
|
-
// Marker-only: a constant flag value, not the target. The body
|
|
1047
|
-
// is the source of truth for the target (no 2 KiB cap, no
|
|
1048
|
-
// header-encoding considerations). See SYMLINK_TARGET_META_KEY
|
|
1049
|
-
// doc for the full reasoning.
|
|
1050
|
-
[SYMLINK_TARGET_META_KEY]: SYMLINK_MARKER_META_VALUE,
|
|
1051
|
-
...(author ? buildAuthorMetadata(author, createdAt) : {}),
|
|
1052
|
-
};
|
|
1053
|
-
|
|
1054
|
-
const response = await io.putObject({
|
|
1055
|
-
key,
|
|
1056
|
-
// Body = SYMLINK_BODY_PREFIX + target (UTF-8). The prefix is what
|
|
1057
|
-
// makes a symlink record's ETag distinguishable from a regular
|
|
1058
|
-
// file whose contents happen to equal the target string — the
|
|
1059
|
-
// LIST-based pull planner can't see per-object metadata, so ETag
|
|
1060
|
-
// is its only drift signal across symlink ↔ regular-file
|
|
1061
|
-
// transitions. See SYMLINK_BODY_PREFIX doc above.
|
|
1062
|
-
body: symlinkBody,
|
|
1063
|
-
contentType: "application/octet-stream",
|
|
1064
|
-
metadata: Metadata,
|
|
1065
|
-
...(precondition ?? {}),
|
|
1066
|
-
});
|
|
1067
|
-
|
|
1068
|
-
return { etag: response.etag };
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
/**
|
|
1072
|
-
* Download an object to localPath and return its S3 user-metadata.
|
|
1073
|
-
*
|
|
1074
|
-
* Materializes regular files and symlink records (the symlink branch
|
|
1075
|
-
* reconstructs the link from the body/marker). The GetObject response
|
|
1076
|
-
* already carries `response.Metadata` (S3 lowercases keys), so we
|
|
1077
|
-
* return it to callers — e.g. the pull loop reads `created-by` to
|
|
1078
|
-
* attribute downloaded files to their author with zero extra network.
|
|
1079
|
-
*/
|
|
1080
|
-
export interface DownloadModeWarning {
|
|
1081
|
-
/** The mode guardrail that could not be applied exactly. */
|
|
1082
|
-
reason: "missing-hq-mode" | "invalid-hq-mode" | "chmod-failed";
|
|
1083
|
-
/** Legacy objects can retain a local file's known-good permission bits. */
|
|
1084
|
-
fallback?: "preserved-local-mode" | "receiver-default";
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
/**
|
|
1088
|
-
* A download whose parent directory exists as a DANGLING symlink.
|
|
1089
|
-
*
|
|
1090
|
-
* The vault stores directory symlinks as first-class objects (e.g.
|
|
1091
|
-
* `companies/<co>/.obsidian -> ../../.obsidian`) AND stores files beneath them.
|
|
1092
|
-
* When the link's target is absent on this machine, the child write fails —
|
|
1093
|
-
* and it fails as ENOENT rather than EEXIST, because `mkdir(2)` returns EEXIST
|
|
1094
|
-
* for the link and Node's `recursive: true` implementation then stats it, which
|
|
1095
|
-
* fails on a dangling link. Raw, that surfaced as a hard error that marked the
|
|
1096
|
-
* WHOLE company `errored`/`partial` on every sync cycle (2026-07-24 dogfood box:
|
|
1097
|
-
* `.obsidian/hotkeys.json: ENOENT ... mkdir '.../companies/indigo/.obsidian'`).
|
|
1098
|
-
*
|
|
1099
|
-
* It is a per-object condition, not a company-level failure, so it is typed and
|
|
1100
|
-
* the pull loop skips the object loudly instead of failing the run. We do NOT
|
|
1101
|
-
* materialize the link target: the target is by definition outside the
|
|
1102
|
-
* directory being synced, and creating it would write through a path the
|
|
1103
|
-
* caller's containment guard deliberately refuses.
|
|
1104
|
-
*/
|
|
1105
|
-
export class DanglingSymlinkParentError extends Error {
|
|
1106
|
-
readonly key: string;
|
|
1107
|
-
readonly dir: string;
|
|
1108
|
-
constructor(key: string, dir: string) {
|
|
1109
|
-
super(
|
|
1110
|
-
`download skipped: parent directory is a dangling symlink (${dir}) — ` +
|
|
1111
|
-
`the vault stores it as a link whose target is absent locally`,
|
|
1112
|
-
);
|
|
1113
|
-
this.name = "DanglingSymlinkParentError";
|
|
1114
|
-
this.key = key;
|
|
1115
|
-
this.dir = dir;
|
|
1116
|
-
}
|
|
1117
|
-
}
|
|
1118
|
-
|
|
1119
|
-
/**
|
|
1120
|
-
* Create the parent directory for a download, distinguishing the
|
|
1121
|
-
* dangling-symlink case from a genuine mkdir failure.
|
|
1122
|
-
*
|
|
1123
|
-
* Callers must have already established that `dir` does not resolve
|
|
1124
|
-
* (`fs.existsSync(dir) === false`); `existsSync` FOLLOWS symlinks, so a
|
|
1125
|
-
* dangling link reports absent here while `lstat` still sees the link.
|
|
1126
|
-
*/
|
|
1127
|
-
function ensureDownloadParentDir(dir: string, key: string): void {
|
|
1128
|
-
let link: fs.Stats | null = null;
|
|
1129
|
-
try {
|
|
1130
|
-
link = fs.lstatSync(dir);
|
|
1131
|
-
} catch {
|
|
1132
|
-
// Genuinely absent — the ordinary first-write path.
|
|
1133
|
-
}
|
|
1134
|
-
if (link?.isSymbolicLink()) {
|
|
1135
|
-
throw new DanglingSymlinkParentError(key, dir);
|
|
1136
|
-
}
|
|
1137
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
1138
|
-
}
|
|
1139
|
-
|
|
1140
|
-
export async function downloadFile(
|
|
1141
|
-
ctx: EntityContext,
|
|
1142
|
-
key: string,
|
|
1143
|
-
localPath: string,
|
|
1144
|
-
options: {
|
|
1145
|
-
beforeReplace?: () => void;
|
|
1146
|
-
/**
|
|
1147
|
-
* Platform override so win32 symlink semantics are directly testable
|
|
1148
|
-
* on any host (same convention as local-path-codec.ts).
|
|
1149
|
-
*/
|
|
1150
|
-
win32?: boolean;
|
|
1151
|
-
} = {},
|
|
1152
|
-
): Promise<{
|
|
1153
|
-
metadata?: Record<string, string>;
|
|
1154
|
-
contentHash?: string;
|
|
1155
|
-
contentSize?: number;
|
|
1156
|
-
/** Non-fatal mode guardrail warnings for caller telemetry. */
|
|
1157
|
-
modeWarnings?: DownloadModeWarning[];
|
|
1158
|
-
}> {
|
|
1159
|
-
const io = resolveObjectIO(ctx);
|
|
1160
|
-
|
|
1161
|
-
const { body, metadata } = await getObjectStream(io, key);
|
|
1162
|
-
const iterator = body[Symbol.asyncIterator]();
|
|
1163
|
-
const initialChunks: Buffer[] = [];
|
|
1164
|
-
let initialLength = 0;
|
|
1165
|
-
while (initialLength < SYMLINK_BODY_PREFIX_BYTES.length) {
|
|
1166
|
-
const next = await iterator.next();
|
|
1167
|
-
if (next.done) break;
|
|
1168
|
-
const chunk = bufferFromChunk(next.value);
|
|
1169
|
-
if (chunk.length === 0) continue;
|
|
1170
|
-
initialChunks.push(chunk);
|
|
1171
|
-
initialLength += chunk.length;
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
const dir = path.dirname(localPath);
|
|
1175
|
-
if (!fs.existsSync(dir)) {
|
|
1176
|
-
ensureDownloadParentDir(dir, key);
|
|
1177
|
-
}
|
|
1178
|
-
|
|
1179
|
-
// Symlink path: presence of SYMLINK_TARGET_META_KEY (any non-empty
|
|
1180
|
-
// value) is the discriminator. The TARGET is now sourced from the
|
|
1181
|
-
// body — the marker-only metadata convention removes the 2 KiB
|
|
1182
|
-
// header limit so long POSIX targets don't fail PutObject.
|
|
1183
|
-
//
|
|
1184
|
-
// S3 lowercases user-metadata keys on read (and sometimes on
|
|
1185
|
-
// write), so the lookup uses the lowercased form. We don't
|
|
1186
|
-
// normalize Metadata keys ourselves — the AWS SDK already does it.
|
|
1187
|
-
const symlinkMarker = metadata?.[SYMLINK_TARGET_META_KEY];
|
|
1188
|
-
// Discriminator: the metadata marker is the primary signal, but the body
|
|
1189
|
-
// prefix is a header-loss fallback (S3 cross-region replication of data
|
|
1190
|
-
// only, a console copy that drops Metadata, a metadata-stripping transport,
|
|
1191
|
-
// or a poisoned regular-file re-upload of a sentinel). Honor BOTH — a
|
|
1192
|
-
// marker-less object whose body starts with SYMLINK_BODY_PREFIX still
|
|
1193
|
-
// rematerializes as a link instead of being written out as plain
|
|
1194
|
-
// `hq-symlink:<target>` text, which would poison the key on the next push.
|
|
1195
|
-
// The body is already buffered (tiny for symlink records); reading it here
|
|
1196
|
-
// is behavior-preserving for regular files (whose body never starts with
|
|
1197
|
-
// the prefix per the SYMLINK_BODY_PREFIX doc). See SYMLINK_BODY_PREFIX.
|
|
1198
|
-
const leading = leadingBytes(
|
|
1199
|
-
initialChunks,
|
|
1200
|
-
initialLength,
|
|
1201
|
-
SYMLINK_BODY_PREFIX_BYTES.length,
|
|
1202
|
-
);
|
|
1203
|
-
const bodyHasSymlinkPrefix =
|
|
1204
|
-
leading.length >= SYMLINK_BODY_PREFIX_BYTES.length &&
|
|
1205
|
-
leading.subarray(0, SYMLINK_BODY_PREFIX_BYTES.length).equals(
|
|
1206
|
-
SYMLINK_BODY_PREFIX_BYTES,
|
|
1207
|
-
);
|
|
1208
|
-
const isSymlinkRecord =
|
|
1209
|
-
(typeof symlinkMarker === "string" && symlinkMarker.length > 0) ||
|
|
1210
|
-
bodyHasSymlinkPrefix;
|
|
1211
|
-
if (isSymlinkRecord) {
|
|
1212
|
-
const objectBody = await collectRemainingChunks(initialChunks, iterator);
|
|
1213
|
-
// The target lives in the body (marker-only metadata convention).
|
|
1214
|
-
// Symlink record bodies are bounded by target length (typically
|
|
1215
|
-
// <300 bytes for relative paths, hard-capped by S3's 5 GB object
|
|
1216
|
-
// size); only this branch buffers the body so regular files can stream.
|
|
1217
|
-
let symlinkTarget: string;
|
|
1218
|
-
if (bodyHasSymlinkPrefix) {
|
|
1219
|
-
symlinkTarget = objectBody.toString(
|
|
1220
|
-
"utf-8",
|
|
1221
|
-
SYMLINK_BODY_PREFIX_BYTES.length,
|
|
1222
|
-
);
|
|
1223
|
-
} else {
|
|
1224
|
-
// Backward-compat fallback: a legacy upload from earlier in
|
|
1225
|
-
// this PR's lifetime stored the target in metadata (raw or
|
|
1226
|
-
// base64) rather than the body. decodeSymlinkMetadataValue
|
|
1227
|
-
// round-trip-validates so a raw value passes through and a
|
|
1228
|
-
// base64 value decodes; either way we get the target. This branch
|
|
1229
|
-
// is only reachable when the body lacks the prefix, which (given
|
|
1230
|
-
// isSymlinkRecord) means the marker is present — the `?? ""` is a
|
|
1231
|
-
// type guard for that invariant, and the length-0 check below
|
|
1232
|
-
// catches the impossible empty case rather than passing it on.
|
|
1233
|
-
symlinkTarget = decodeSymlinkMetadataValue(symlinkMarker ?? "");
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
if (symlinkTarget.length === 0) {
|
|
1237
|
-
throw new Error(
|
|
1238
|
-
`Symlink record for ${key} had no target (body: ${objectBody.length} bytes, marker: ${symlinkMarker})`,
|
|
1239
|
-
);
|
|
1240
|
-
}
|
|
1241
|
-
|
|
1242
|
-
// Idempotence fast path: the .claude/skills link farm makes symlink
|
|
1243
|
-
// records the most re-pulled object class, and a pull that changes
|
|
1244
|
-
// nothing must not touch disk at all (pre-fix, an already-correct
|
|
1245
|
-
// Windows dir link still went through temp+rename, EPERM'd every
|
|
1246
|
-
// pass, never journaled, and re-downloaded forever). On POSIX a
|
|
1247
|
-
// target-string match is sufficient (lstat + readlink — never stat:
|
|
1248
|
-
// dangling links are legitimate). On win32 a matching target string is
|
|
1249
|
-
// NOT sufficient — links are flavored, and a file-type link at a
|
|
1250
|
-
// directory target (minted by an older client, or while the target was
|
|
1251
|
-
// absent) is broken despite the matching string. There the fast path
|
|
1252
|
-
// additionally requires stat-follow to confirm the link FUNCTIONS as
|
|
1253
|
-
// the type win32SymlinkType() would mint now; anything unconfirmable
|
|
1254
|
-
// (including a dangling link, whose flavor cannot be probed) falls
|
|
1255
|
-
// through to replacement, which is cheap and journals correctly.
|
|
1256
|
-
const win32 = options.win32 ?? process.platform === "win32";
|
|
1257
|
-
let existingLink: fs.Stats | null = null;
|
|
1258
|
-
try {
|
|
1259
|
-
existingLink = fs.lstatSync(localPath);
|
|
1260
|
-
} catch {
|
|
1261
|
-
// Absent (or unreadable) destination: no fast path; the normal
|
|
1262
|
-
// materialization below surfaces real filesystem faults loudly.
|
|
1263
|
-
}
|
|
1264
|
-
if (existingLink?.isSymbolicLink()) {
|
|
1265
|
-
let existingTarget: string | null = null;
|
|
1266
|
-
try {
|
|
1267
|
-
existingTarget = fs.readlinkSync(localPath);
|
|
1268
|
-
} catch {
|
|
1269
|
-
// Link raced away between lstat and readlink; replace it below.
|
|
1270
|
-
}
|
|
1271
|
-
if (
|
|
1272
|
-
existingTarget !== null &&
|
|
1273
|
-
symlinkTargetsEqual(existingTarget, symlinkTarget, localPath, win32)
|
|
1274
|
-
) {
|
|
1275
|
-
if (!win32) return { metadata };
|
|
1276
|
-
const desiredIsDirectory = win32StagedLinkIsDirectoryFlavored(
|
|
1277
|
-
symlinkTarget,
|
|
1278
|
-
localPath,
|
|
1279
|
-
);
|
|
1280
|
-
let followed: fs.Stats | null = null;
|
|
1281
|
-
try {
|
|
1282
|
-
followed = fs.statSync(localPath);
|
|
1283
|
-
} catch {
|
|
1284
|
-
// Unresolvable via this link (dangling, or a wrong-flavor link
|
|
1285
|
-
// that Windows refuses to traverse): recreate below.
|
|
1286
|
-
}
|
|
1287
|
-
if (followed !== null && followed.isDirectory() === desiredIsDirectory) {
|
|
1288
|
-
return { metadata };
|
|
1289
|
-
}
|
|
1290
|
-
}
|
|
1291
|
-
}
|
|
1292
|
-
|
|
1293
|
-
const tempPath = downloadTempPath(localPath);
|
|
1294
|
-
try {
|
|
1295
|
-
// Creation goes through createStagedSymlink (#219): on win32 a directory
|
|
1296
|
-
// overlay becomes an unprivileged JUNCTION rather than a 'dir' NTFS
|
|
1297
|
-
// symlink, which is what stops the EPERM that pushed the one-shot runner
|
|
1298
|
-
// to PARTIAL_SYNC_EXIT. The staged link lands beside `localPath`, so its
|
|
1299
|
-
// flavor decision resolves the target identically to the fast path above.
|
|
1300
|
-
createStagedSymlink(symlinkTarget, tempPath, stagedSymlinkOpsFor(win32));
|
|
1301
|
-
options.beforeReplace?.();
|
|
1302
|
-
replaceStagedPath(tempPath, localPath);
|
|
1303
|
-
} catch (err) {
|
|
1304
|
-
removeTempPath(tempPath);
|
|
1305
|
-
throw err;
|
|
1306
|
-
}
|
|
1307
|
-
return { metadata };
|
|
1308
|
-
}
|
|
1309
|
-
const tempPath = downloadTempPath(localPath);
|
|
1310
|
-
// A legacy object has no hq-mode. If it is replacing a regular local file,
|
|
1311
|
-
// that local mode is the only trustworthy signal we have — retaining it is
|
|
1312
|
-
// strictly safer than replacing (for example) a 0755 hook with the receiver
|
|
1313
|
-
// umask's usual 0644. Never inspect a symlink or infer intent from content.
|
|
1314
|
-
let existingLocalMode: number | undefined;
|
|
1315
|
-
try {
|
|
1316
|
-
const existing = fs.lstatSync(localPath);
|
|
1317
|
-
if (existing.isFile()) existingLocalMode = existing.mode & 0o777;
|
|
1318
|
-
} catch {
|
|
1319
|
-
// Fresh destination / inaccessible prior path: no local fallback.
|
|
1320
|
-
}
|
|
1321
|
-
let tempReady = true;
|
|
1322
|
-
let streamed: { hash: string; size: number };
|
|
1323
|
-
const modeWarnings: DownloadModeWarning[] = [];
|
|
1324
|
-
try {
|
|
1325
|
-
streamed = await streamRegularFileToTemp(tempPath, initialChunks, iterator);
|
|
1326
|
-
|
|
1327
|
-
// Bug #5 — apply source-side mode after the byte write. See
|
|
1328
|
-
// FILE_MODE_META_KEY for the metadata contract. The staged path is a
|
|
1329
|
-
// regular file, then it is atomically renamed over the destination.
|
|
1330
|
-
//
|
|
1331
|
-
// Codex P2 (PR #24 round 3): strict octal-only regex BEFORE parseInt.
|
|
1332
|
-
// parseInt(modeOctal, 8) accepts partial-prefix garbage — "755junk"
|
|
1333
|
-
// parses to 0o755 instead of NaN — so tampered or malformed metadata
|
|
1334
|
-
// could still change local permissions unexpectedly. The regex
|
|
1335
|
-
// requires 1–4 pure octal digits (`[0-7]{1,4}$`), which matches what
|
|
1336
|
-
// the upload side stamps (`(mode & 0o777).toString(8)` → at most
|
|
1337
|
-
// three digits, all 0–7) and rejects everything else.
|
|
1338
|
-
const modeOctal = metadata?.[FILE_MODE_META_KEY];
|
|
1339
|
-
let modeToApply: number | undefined;
|
|
1340
|
-
if (typeof modeOctal === "string" && /^[0-7]{1,4}$/.test(modeOctal)) {
|
|
1341
|
-
const parsed = parseInt(modeOctal, 8);
|
|
1342
|
-
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 0o777) {
|
|
1343
|
-
modeToApply = parsed;
|
|
1344
|
-
}
|
|
1345
|
-
} else if (modeOctal === undefined) {
|
|
1346
|
-
modeToApply = existingLocalMode;
|
|
1347
|
-
modeWarnings.push({
|
|
1348
|
-
reason: "missing-hq-mode",
|
|
1349
|
-
fallback:
|
|
1350
|
-
existingLocalMode === undefined
|
|
1351
|
-
? "receiver-default"
|
|
1352
|
-
: "preserved-local-mode",
|
|
1353
|
-
});
|
|
1354
|
-
} else {
|
|
1355
|
-
modeWarnings.push({ reason: "invalid-hq-mode", fallback: "receiver-default" });
|
|
1356
|
-
}
|
|
1357
|
-
|
|
1358
|
-
if (modeToApply !== undefined) {
|
|
1359
|
-
try {
|
|
1360
|
-
fs.chmodSync(tempPath, modeToApply);
|
|
1361
|
-
} catch {
|
|
1362
|
-
// chmod failure (read-only FS, EPERM) is non-fatal — the file is
|
|
1363
|
-
// materialized, but callers must receive a telemetry signal instead
|
|
1364
|
-
// of silently believing the guardrail held.
|
|
1365
|
-
modeWarnings.push({ reason: "chmod-failed" });
|
|
1366
|
-
}
|
|
1367
|
-
}
|
|
1368
|
-
|
|
1369
|
-
// 5.37.0 — apply source-side mtime after the byte write (and after the
|
|
1370
|
-
// chmod above; ordering between chmod and utimes doesn't matter, but
|
|
1371
|
-
// both must run AFTER writeFileSync because writeFileSync resets mtime
|
|
1372
|
-
// to wall-clock-now). See FILE_MTIME_META_KEY for the metadata contract.
|
|
1373
|
-
//
|
|
1374
|
-
// Strict-numeric regex BEFORE parseInt — same Codex P2 lesson as hq-mode.
|
|
1375
|
-
// `^-?[0-9]{1,16}$` rejects partial-prefix garbage ("175junk" → 175),
|
|
1376
|
-
// empty, double-signed, decimals, whitespace, and oversized strings. A
|
|
1377
|
-
// single optional leading `-` is allowed so pre-epoch / reproducible-build
|
|
1378
|
-
// timestamps round-trip (Codex PR #27 P2 — `mtimeMs === 0` and negative
|
|
1379
|
-
// epoch values are legitimate). 16 digits comfortably covers any plausible
|
|
1380
|
-
// epoch-ms value (year ~5138 is 16 digits; we'll cross that bridge later).
|
|
1381
|
-
//
|
|
1382
|
-
// The staged file is regular; rename preserves these stamps at localPath.
|
|
1383
|
-
//
|
|
1384
|
-
// Composition with the 5.36.0 lstat fast-path: the journal stamp at the
|
|
1385
|
-
// share/sync call sites runs AFTER downloadFile returns, so the lstat
|
|
1386
|
-
// it captures sees the post-utimes mtime. Verified in cli/sync.ts
|
|
1387
|
-
// (downloadFile → lstatSync → updateEntry) and cli/share.ts (pull path
|
|
1388
|
-
// similarly lstats after downloadFile). If a future caller stamps the
|
|
1389
|
-
// journal BEFORE downloadFile completes, the fast-path will stale and
|
|
1390
|
-
// re-hash every sync forever — keep the call-site invariant intact.
|
|
1391
|
-
const mtimeRaw = metadata?.[FILE_MTIME_META_KEY];
|
|
1392
|
-
if (typeof mtimeRaw === "string" && /^-?[0-9]{1,16}$/.test(mtimeRaw)) {
|
|
1393
|
-
const mtimeMs = parseInt(mtimeRaw, 10);
|
|
1394
|
-
if (Number.isFinite(mtimeMs)) {
|
|
1395
|
-
try {
|
|
1396
|
-
// utimesSync accepts seconds OR Date; use Date(ms) for precision.
|
|
1397
|
-
// atime = mtime is fine — many filesystems are mounted noatime
|
|
1398
|
-
// and distinguishing "access" vs "modification" time doesn't
|
|
1399
|
-
// matter for sync semantics. Setting both keeps the on-disk
|
|
1400
|
-
// state deterministic across receivers.
|
|
1401
|
-
const mtimeDate = new Date(mtimeMs);
|
|
1402
|
-
fs.utimesSync(tempPath, mtimeDate, mtimeDate);
|
|
1403
|
-
} catch {
|
|
1404
|
-
// EPERM / read-only FS / file just unlinked → non-fatal. The
|
|
1405
|
-
// file is materialized at write-time mtime; the source-of-truth
|
|
1406
|
-
// can re-sync next pass.
|
|
1407
|
-
}
|
|
1408
|
-
}
|
|
1409
|
-
}
|
|
1410
|
-
|
|
1411
|
-
options.beforeReplace?.();
|
|
1412
|
-
replaceStagedPath(tempPath, localPath);
|
|
1413
|
-
tempReady = false;
|
|
1414
|
-
} finally {
|
|
1415
|
-
if (tempReady) removeTempPath(tempPath);
|
|
1416
|
-
}
|
|
1417
|
-
|
|
1418
|
-
// TODO: stamp hq-btime once Node lands lbirthtime (or birthtimeSync).
|
|
1419
|
-
// The push side already emits hq-btime when the source FS tracks a
|
|
1420
|
-
// distinct creation time, so a future receiver upgrade picks it up
|
|
1421
|
-
// automatically without a server-side data migration.
|
|
1422
|
-
|
|
1423
|
-
if (modeWarnings.length > 0) {
|
|
1424
|
-
for (const warning of modeWarnings) {
|
|
1425
|
-
console.warn(
|
|
1426
|
-
`[hq-sync] mode warning for ${key}: ${warning.reason}` +
|
|
1427
|
-
(warning.fallback ? ` (${warning.fallback})` : ""),
|
|
1428
|
-
);
|
|
1429
|
-
}
|
|
1430
|
-
}
|
|
1431
|
-
|
|
1432
|
-
return {
|
|
1433
|
-
metadata,
|
|
1434
|
-
contentHash: streamed.hash,
|
|
1435
|
-
contentSize: streamed.size,
|
|
1436
|
-
...(modeWarnings.length > 0 ? { modeWarnings } : {}),
|
|
1437
|
-
};
|
|
1438
|
-
}
|
|
1439
|
-
|
|
1440
|
-
export interface RemoteFile {
|
|
1441
|
-
key: string;
|
|
1442
|
-
size: number;
|
|
1443
|
-
lastModified: Date;
|
|
1444
|
-
etag: string;
|
|
1445
|
-
}
|
|
1446
|
-
|
|
1447
|
-
export async function listRemoteFiles(
|
|
1448
|
-
ctx: EntityContext,
|
|
1449
|
-
prefix?: string,
|
|
1450
|
-
): Promise<RemoteFile[]> {
|
|
1451
|
-
const io = resolveObjectIO(ctx);
|
|
1452
|
-
const files: RemoteFile[] = [];
|
|
1453
|
-
let continuationToken: string | undefined;
|
|
1454
|
-
|
|
1455
|
-
do {
|
|
1456
|
-
const page = await io.listObjects({ prefix, continuationToken });
|
|
1457
|
-
|
|
1458
|
-
for (const obj of page.objects) {
|
|
1459
|
-
// Drop S3 directory-marker objects: the canonical shape is `0-byte
|
|
1460
|
-
// key ending in '/'` (S3 console "Create folder", `aws s3 sync` of
|
|
1461
|
-
// empty dirs, sync tools that mirror empty trees). Two downstream
|
|
1462
|
-
// sites blow up on them — pull planner (sync.ts: `hashFile` calls
|
|
1463
|
-
// `readFileSync` on an existing local dir → EISDIR "read") and the
|
|
1464
|
-
// download path (s3.ts: `writeFileSync` on a trailing-slash path
|
|
1465
|
-
// → EISDIR "open" after the parent mkdir creates the leaf as a
|
|
1466
|
-
// directory). Filtering here eliminates both.
|
|
1467
|
-
//
|
|
1468
|
-
// Narrow on size===0 (not just trailing-slash) so a hypothetical
|
|
1469
|
-
// non-empty object whose key happens to end in '/' is NOT silently
|
|
1470
|
-
// hidden — it stays visible and downloadFile surfaces the same
|
|
1471
|
-
// EISDIR "open" error pointing at the specific key, which is the
|
|
1472
|
-
// signal an operator needs to reconcile the bucket. The vault
|
|
1473
|
-
// service doesn't have a code path that produces such an object,
|
|
1474
|
-
// but the listing returns whatever lives in the bucket; silent
|
|
1475
|
-
// drop would be worse than loud failure for that case.
|
|
1476
|
-
//
|
|
1477
|
-
// Real 0-byte placeholders like `.gitkeep` never end in `/` and
|
|
1478
|
-
// continue to flow through — the 5.13.0 `.gitkeep` regression
|
|
1479
|
-
// remains fixed. (The `!key` guard now lives in the ObjectIO layer.)
|
|
1480
|
-
if (obj.key.endsWith("/") && obj.size === 0) continue;
|
|
1481
|
-
|
|
1482
|
-
files.push({
|
|
1483
|
-
key: obj.key,
|
|
1484
|
-
size: obj.size,
|
|
1485
|
-
lastModified: obj.lastModified,
|
|
1486
|
-
etag: obj.etag,
|
|
1487
|
-
});
|
|
1488
|
-
}
|
|
1489
|
-
|
|
1490
|
-
continuationToken = page.nextContinuationToken;
|
|
1491
|
-
} while (continuationToken);
|
|
1492
|
-
|
|
1493
|
-
return files;
|
|
1494
|
-
}
|
|
1495
|
-
|
|
1496
|
-
export async function deleteRemoteFile(
|
|
1497
|
-
ctx: EntityContext,
|
|
1498
|
-
key: string,
|
|
1499
|
-
): Promise<void> {
|
|
1500
|
-
await resolveObjectIO(ctx).deleteObject(key);
|
|
1501
|
-
}
|
|
1502
|
-
|
|
1503
|
-
/**
|
|
1504
|
-
* Check if a remote key exists and return its metadata.
|
|
1505
|
-
*/
|
|
1506
|
-
export async function headRemoteFile(
|
|
1507
|
-
ctx: EntityContext,
|
|
1508
|
-
key: string,
|
|
1509
|
-
): Promise<{ lastModified: Date; etag: string; size: number; metadata?: Record<string, string> } | null> {
|
|
1510
|
-
return resolveObjectIO(ctx).headObject(key);
|
|
1511
|
-
}
|
|
1512
|
-
|
|
1513
|
-
function getMimeType(filePath: string): string {
|
|
1514
|
-
const ext = path.extname(filePath).toLowerCase();
|
|
1515
|
-
const mimeTypes: Record<string, string> = {
|
|
1516
|
-
".md": "text/markdown",
|
|
1517
|
-
".json": "application/json",
|
|
1518
|
-
".yaml": "text/yaml",
|
|
1519
|
-
".yml": "text/yaml",
|
|
1520
|
-
".ts": "text/typescript",
|
|
1521
|
-
".js": "text/javascript",
|
|
1522
|
-
".txt": "text/plain",
|
|
1523
|
-
".html": "text/html",
|
|
1524
|
-
".css": "text/css",
|
|
1525
|
-
".png": "image/png",
|
|
1526
|
-
".jpg": "image/jpeg",
|
|
1527
|
-
".jpeg": "image/jpeg",
|
|
1528
|
-
".svg": "image/svg+xml",
|
|
1529
|
-
".pdf": "application/pdf",
|
|
1530
|
-
};
|
|
1531
|
-
return mimeTypes[ext] || "application/octet-stream";
|
|
1532
|
-
}
|