@indigoai-us/hq-cloud 5.36.0 → 5.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +128 -1
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +256 -1
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +35 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +10 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/s3.d.ts +54 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +150 -1
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +261 -1
- package/dist/s3.test.js.map +1 -1
- package/package.json +1 -1
- package/src/cli/share.test.ts +311 -1
- package/src/cli/share.ts +146 -2
- package/src/cli/sync.ts +46 -1
- package/src/s3.test.ts +296 -0
- package/src/s3.ts +155 -1
package/src/s3.test.ts
CHANGED
|
@@ -88,6 +88,8 @@ import {
|
|
|
88
88
|
SYMLINK_MARKER_META_VALUE,
|
|
89
89
|
encodeSymlinkMetadataValue,
|
|
90
90
|
decodeSymlinkMetadataValue,
|
|
91
|
+
FILE_MTIME_META_KEY,
|
|
92
|
+
FILE_BTIME_META_KEY,
|
|
91
93
|
} from "./s3.js";
|
|
92
94
|
import type { EntityContext } from "./types.js";
|
|
93
95
|
|
|
@@ -219,6 +221,66 @@ describe("uploadFile", () => {
|
|
|
219
221
|
}
|
|
220
222
|
});
|
|
221
223
|
|
|
224
|
+
it("stamps source file mtimeMs as hq-mtime metadata (5.37.0 — preserve mtime)", async () => {
|
|
225
|
+
// Symmetric to Bug #5's mode preservation. A file's modification time
|
|
226
|
+
// should follow it across machines instead of resetting to "the time of
|
|
227
|
+
// sync." Push stamps `hq-mtime` = integer-ms epoch string; pull applies
|
|
228
|
+
// it via utimesSync. See FILE_MTIME_META_KEY doc.
|
|
229
|
+
const knownMs = 1_700_000_000_000; // 2023-11-14T22:13:20Z, comfortably finite
|
|
230
|
+
const knownDate = new Date(knownMs);
|
|
231
|
+
fs.utimesSync(tmpFile, knownDate, knownDate);
|
|
232
|
+
|
|
233
|
+
await uploadFile(makeCtx(), tmpFile, "stamped.md");
|
|
234
|
+
|
|
235
|
+
const put = sentCommands.find((c) => c.name === "PutObjectCommand");
|
|
236
|
+
expect(put).toBeDefined();
|
|
237
|
+
const meta = put!.input.Metadata as Record<string, string> | undefined;
|
|
238
|
+
expect(meta?.[FILE_MTIME_META_KEY]).toBe(String(knownMs));
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
it("stamps hq-btime only when birthtime is supported AND distinct from mtime", async () => {
|
|
242
|
+
// Many filesystems (Linux/ext4 historically, tmpfs, some FUSE mounts)
|
|
243
|
+
// return birthtimeMs === 0 or birthtimeMs === mtimeMs. Stamping in
|
|
244
|
+
// those cases would be pure noise; the upload side filters it out.
|
|
245
|
+
// We don't control the test filesystem's birthtime semantics, so the
|
|
246
|
+
// assertion is shape-only: IF hq-btime is present, it's a numeric
|
|
247
|
+
// string AND it's distinct from hq-mtime.
|
|
248
|
+
await uploadFile(makeCtx(), tmpFile, "btime.md");
|
|
249
|
+
|
|
250
|
+
const put = sentCommands.find((c) => c.name === "PutObjectCommand");
|
|
251
|
+
const meta = put!.input.Metadata as Record<string, string> | undefined;
|
|
252
|
+
const btime = meta?.[FILE_BTIME_META_KEY];
|
|
253
|
+
const mtime = meta?.[FILE_MTIME_META_KEY];
|
|
254
|
+
if (typeof btime === "string") {
|
|
255
|
+
expect(btime).toMatch(/^[0-9]{1,16}$/);
|
|
256
|
+
expect(btime).not.toBe(mtime);
|
|
257
|
+
}
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
it("does NOT stamp hq-mtime / hq-btime for symlinks (uploadFile branch)", async () => {
|
|
261
|
+
// uploadFile is the regular-file path; uploadSymlink is the symlink
|
|
262
|
+
// path. uploadFile's lstat.isSymbolicLink() guard skips ALL three of
|
|
263
|
+
// mode + mtime + btime stamping for symlinks, mirroring the existing
|
|
264
|
+
// hq-mode policy: symlink times are OS-controlled and fs.utimesSync
|
|
265
|
+
// would follow the link and mutate the target. The link is set up
|
|
266
|
+
// pointing at a real target so fs.readFileSync (which follows links)
|
|
267
|
+
// doesn't ENOENT before the metadata stamping branch even runs.
|
|
268
|
+
const targetPath = path.join(os.tmpdir(), `s3-link-target-${Date.now()}-${Math.random()}.txt`);
|
|
269
|
+
fs.writeFileSync(targetPath, "real-bytes");
|
|
270
|
+
const linkPath = path.join(os.tmpdir(), `s3-link-test-${Date.now()}-${Math.random()}.lnk`);
|
|
271
|
+
fs.symlinkSync(targetPath, linkPath);
|
|
272
|
+
|
|
273
|
+
await uploadFile(makeCtx(), linkPath, "link.md");
|
|
274
|
+
|
|
275
|
+
const put = sentCommands.find((c) => c.name === "PutObjectCommand");
|
|
276
|
+
const meta = put!.input.Metadata as Record<string, string> | undefined;
|
|
277
|
+
expect(meta?.[FILE_MTIME_META_KEY]).toBeUndefined();
|
|
278
|
+
expect(meta?.[FILE_BTIME_META_KEY]).toBeUndefined();
|
|
279
|
+
|
|
280
|
+
fs.unlinkSync(linkPath);
|
|
281
|
+
fs.unlinkSync(targetPath);
|
|
282
|
+
});
|
|
283
|
+
|
|
222
284
|
it("elides non-ASCII or empty author fields rather than throwing", async () => {
|
|
223
285
|
// S3 user-defined metadata must be ASCII-only and total ≤ 2KB. Partial
|
|
224
286
|
// attribution beats hard failure — values that fail the printable check
|
|
@@ -770,6 +832,240 @@ describe("downloadFile", () => {
|
|
|
770
832
|
expect(fs.readFileSync(localPath, "utf-8")).toBe("legacy");
|
|
771
833
|
});
|
|
772
834
|
|
|
835
|
+
it("applies hq-mtime via utimesSync after byte write (5.37.0 — preserve mtime)", async () => {
|
|
836
|
+
// Round-trip pair to the s3.upload mtime stamp test. The receiver must
|
|
837
|
+
// set the local file's mtime to the metadata-stamped value AFTER the
|
|
838
|
+
// byte write, mirroring the chmod-after-write shape from Bug #5.
|
|
839
|
+
const knownMs = 1_700_000_000_000;
|
|
840
|
+
nextGetObjectResponse = {
|
|
841
|
+
Body: (async function* () {
|
|
842
|
+
yield new Uint8Array([116, 105, 109, 101]); // "time"
|
|
843
|
+
})(),
|
|
844
|
+
Metadata: { [FILE_MTIME_META_KEY]: String(knownMs) },
|
|
845
|
+
};
|
|
846
|
+
|
|
847
|
+
const localPath = path.join(tmpRoot, "mtime-applied.bin");
|
|
848
|
+
await downloadFile(makeCtx(), "mtime-applied.bin", localPath);
|
|
849
|
+
|
|
850
|
+
const stat = fs.statSync(localPath);
|
|
851
|
+
// ±50ms tolerance accommodates ms-precision filesystems and the
|
|
852
|
+
// utimesSync→stat round-trip rounding. macOS APFS reports full ms;
|
|
853
|
+
// some Linux filesystems round to seconds — guard with tolerance.
|
|
854
|
+
expect(Math.abs(stat.mtimeMs - knownMs)).toBeLessThanOrEqual(50);
|
|
855
|
+
});
|
|
856
|
+
|
|
857
|
+
it("rejects malformed hq-mtime metadata via strict-numeric regex", async () => {
|
|
858
|
+
// Symmetric to the hq-mode Codex P2 lesson: parseInt accepts partial-
|
|
859
|
+
// prefix garbage ("175junk" → 175). A strict `^[0-9]{1,16}$` regex
|
|
860
|
+
// BEFORE parseInt rejects empty / signed / decimals / whitespace /
|
|
861
|
+
// oversized / non-numeric and falls through to "no utimes call,"
|
|
862
|
+
// leaving the file at write-time mtime — the legacy back-compat path.
|
|
863
|
+
// NOTE: `-100` is intentionally NOT in this list — Codex PR #27 P2 pointed
|
|
864
|
+
// out that legitimate epoch values include 0 (Unix epoch) and negatives
|
|
865
|
+
// (e.g. reproducible-build clamping pre-1970). Negative-but-well-formed
|
|
866
|
+
// strings are now accepted; see the dedicated round-trip tests below.
|
|
867
|
+
const malformed = [
|
|
868
|
+
"175junk", // trailing garbage — parseInt parses 175 pre-fix
|
|
869
|
+
"abc", // non-numeric
|
|
870
|
+
"", // empty
|
|
871
|
+
" 100 ", // whitespace
|
|
872
|
+
"--100", // double-sign — only ONE optional leading "-" allowed
|
|
873
|
+
"+100", // explicit plus rejected (regex only allows optional "-")
|
|
874
|
+
"3.14", // decimal
|
|
875
|
+
"99999999999999999", // >16 digits
|
|
876
|
+
];
|
|
877
|
+
for (const bad of malformed) {
|
|
878
|
+
// Establish a known pre-write mtime by writing the file first.
|
|
879
|
+
const localPath = path.join(tmpRoot, `bad-mtime-${malformed.indexOf(bad)}.bin`);
|
|
880
|
+
nextGetObjectResponse = {
|
|
881
|
+
Body: (async function* () {
|
|
882
|
+
yield new Uint8Array([98]); // "b"
|
|
883
|
+
})(),
|
|
884
|
+
Metadata: { [FILE_MTIME_META_KEY]: bad },
|
|
885
|
+
};
|
|
886
|
+
const before = Date.now();
|
|
887
|
+
await downloadFile(makeCtx(), `bad-mtime-${malformed.indexOf(bad)}.bin`, localPath);
|
|
888
|
+
const stat = fs.statSync(localPath);
|
|
889
|
+
// The file's mtime MUST be wall-clock-now (write-time), NOT the
|
|
890
|
+
// partial-parse value. For "175junk", that means mtime is in the
|
|
891
|
+
// current millis range, not ms=175 (1970-01-01T00:00:00.175Z).
|
|
892
|
+
// Generic check: mtime is near `before` (within 5s).
|
|
893
|
+
expect(Math.abs(stat.mtimeMs - before)).toBeLessThan(5000);
|
|
894
|
+
// Specific check: never the partial-parse epoch (~175ms after epoch).
|
|
895
|
+
expect(stat.mtimeMs).toBeGreaterThan(1_000_000_000_000);
|
|
896
|
+
}
|
|
897
|
+
});
|
|
898
|
+
|
|
899
|
+
it("round-trips mtimeMs === 0 (Unix epoch — Codex PR #27 P2)", async () => {
|
|
900
|
+
// Codex flagged that filtering `mtimeMs > 0` drops legitimate epoch-0
|
|
901
|
+
// timestamps (and reproducible-build clamping uses exactly 0). The
|
|
902
|
+
// receiver must accept "0", parse it, and stamp `new Date(0)` on disk.
|
|
903
|
+
const sourcePath = path.join(tmpRoot, "epoch-zero-src.bin");
|
|
904
|
+
fs.writeFileSync(sourcePath, "epoch-bytes");
|
|
905
|
+
fs.utimesSync(sourcePath, new Date(0), new Date(0));
|
|
906
|
+
|
|
907
|
+
sentCommands.length = 0;
|
|
908
|
+
await uploadFile(makeCtx(), sourcePath, "epoch-zero.bin");
|
|
909
|
+
const put = sentCommands.find((c) => c.name === "PutObjectCommand");
|
|
910
|
+
const stampedMeta = put!.input.Metadata as Record<string, string>;
|
|
911
|
+
const stampedBody = put!.input.Body as Buffer;
|
|
912
|
+
expect(stampedMeta[FILE_MTIME_META_KEY]).toBe("0");
|
|
913
|
+
|
|
914
|
+
fs.unlinkSync(sourcePath);
|
|
915
|
+
nextGetObjectResponse = {
|
|
916
|
+
Body: (async function* () {
|
|
917
|
+
yield new Uint8Array(stampedBody);
|
|
918
|
+
})(),
|
|
919
|
+
Metadata: stampedMeta,
|
|
920
|
+
};
|
|
921
|
+
await downloadFile(makeCtx(), "epoch-zero.bin", sourcePath);
|
|
922
|
+
|
|
923
|
+
const stat = fs.statSync(sourcePath);
|
|
924
|
+
// 0 is integer-clean — exact match expected, but allow 50ms tolerance
|
|
925
|
+
// for any underlying FS rounding shenanigans.
|
|
926
|
+
expect(Math.abs(stat.mtimeMs - 0)).toBeLessThanOrEqual(50);
|
|
927
|
+
expect(fs.readFileSync(sourcePath, "utf-8")).toBe("epoch-bytes");
|
|
928
|
+
});
|
|
929
|
+
|
|
930
|
+
it("round-trips a negative mtime (pre-epoch — Codex PR #27 P2)", async () => {
|
|
931
|
+
// Some filesystems / reproducible-build pipelines clamp mtime to values
|
|
932
|
+
// before the Unix epoch (negative ms). `new Date(-86400000)` is a legal
|
|
933
|
+
// Date; `fs.utimesSync` accepts it. The receiver MUST NOT drop these.
|
|
934
|
+
const sourceMs = -86_400_000; // 1969-12-31T00:00:00Z, one day before epoch
|
|
935
|
+
const sourcePath = path.join(tmpRoot, "pre-epoch-src.bin");
|
|
936
|
+
fs.writeFileSync(sourcePath, "pre-epoch-bytes");
|
|
937
|
+
fs.utimesSync(sourcePath, new Date(sourceMs), new Date(sourceMs));
|
|
938
|
+
|
|
939
|
+
sentCommands.length = 0;
|
|
940
|
+
await uploadFile(makeCtx(), sourcePath, "pre-epoch.bin");
|
|
941
|
+
const put = sentCommands.find((c) => c.name === "PutObjectCommand");
|
|
942
|
+
const stampedMeta = put!.input.Metadata as Record<string, string>;
|
|
943
|
+
const stampedBody = put!.input.Body as Buffer;
|
|
944
|
+
expect(stampedMeta[FILE_MTIME_META_KEY]).toBe(String(sourceMs));
|
|
945
|
+
|
|
946
|
+
fs.unlinkSync(sourcePath);
|
|
947
|
+
nextGetObjectResponse = {
|
|
948
|
+
Body: (async function* () {
|
|
949
|
+
yield new Uint8Array(stampedBody);
|
|
950
|
+
})(),
|
|
951
|
+
Metadata: stampedMeta,
|
|
952
|
+
};
|
|
953
|
+
await downloadFile(makeCtx(), "pre-epoch.bin", sourcePath);
|
|
954
|
+
|
|
955
|
+
const stat = fs.statSync(sourcePath);
|
|
956
|
+
expect(Math.abs(stat.mtimeMs - sourceMs)).toBeLessThanOrEqual(50);
|
|
957
|
+
expect(fs.readFileSync(sourcePath, "utf-8")).toBe("pre-epoch-bytes");
|
|
958
|
+
});
|
|
959
|
+
|
|
960
|
+
it("accepts well-formed negative hq-mtime (e.g. '-100') on download", async () => {
|
|
961
|
+
// Migration of the old rejection-list case for `-100`. Codex PR #27 P2:
|
|
962
|
+
// negative epoch-ms values are legitimate; the regex now allows a single
|
|
963
|
+
// optional leading `-`, and utimesSync handles it.
|
|
964
|
+
const localPath = path.join(tmpRoot, "negative-mtime.bin");
|
|
965
|
+
nextGetObjectResponse = {
|
|
966
|
+
Body: (async function* () {
|
|
967
|
+
yield new Uint8Array([110]); // "n"
|
|
968
|
+
})(),
|
|
969
|
+
Metadata: { [FILE_MTIME_META_KEY]: "-100" },
|
|
970
|
+
};
|
|
971
|
+
await downloadFile(makeCtx(), "negative-mtime.bin", localPath);
|
|
972
|
+
|
|
973
|
+
const stat = fs.statSync(localPath);
|
|
974
|
+
// mtime should be ~ -100 ms, NOT wall-clock-now. The old code would have
|
|
975
|
+
// left this at write-time; the new code applies the negative stamp.
|
|
976
|
+
expect(Math.abs(stat.mtimeMs - -100)).toBeLessThanOrEqual(50);
|
|
977
|
+
});
|
|
978
|
+
|
|
979
|
+
it("downloads with default mtime when hq-mtime metadata is absent (back-compat)", async () => {
|
|
980
|
+
// Legacy uploads (pre-5.37.0) carry no hq-mtime header. The receiver
|
|
981
|
+
// must NOT crash and must NOT alter the on-disk mtime away from the
|
|
982
|
+
// default `fs.writeFileSync` wall-clock-now behavior. Mirrors the
|
|
983
|
+
// hq-mode back-compat test.
|
|
984
|
+
nextGetObjectResponse = {
|
|
985
|
+
Body: (async function* () {
|
|
986
|
+
yield new Uint8Array([108, 101, 103]); // "leg"
|
|
987
|
+
})(),
|
|
988
|
+
Metadata: {},
|
|
989
|
+
};
|
|
990
|
+
|
|
991
|
+
const before = Date.now();
|
|
992
|
+
const localPath = path.join(tmpRoot, "legacy-mtime.bin");
|
|
993
|
+
await downloadFile(makeCtx(), "legacy-mtime.bin", localPath);
|
|
994
|
+
|
|
995
|
+
const stat = fs.statSync(localPath);
|
|
996
|
+
expect(Math.abs(stat.mtimeMs - before)).toBeLessThan(5000);
|
|
997
|
+
expect(fs.readFileSync(localPath, "utf-8")).toBe("leg");
|
|
998
|
+
});
|
|
999
|
+
|
|
1000
|
+
it("does NOT call utimesSync on the symlink branch (5.37.0)", async () => {
|
|
1001
|
+
// fs.utimesSync follows symlinks (mutates the target's mtime, not the
|
|
1002
|
+
// link's). On the symlink branch we must skip the utimes block entirely
|
|
1003
|
+
// so we don't accidentally mutate an unrelated target file. The
|
|
1004
|
+
// symlink record's hq-mtime, if any, is ignored — symlink mtime is
|
|
1005
|
+
// OS-controlled and Node has no stable lutimes API. Easiest verifier:
|
|
1006
|
+
// the link is materialized AND the test doesn't crash.
|
|
1007
|
+
const targetPath = path.join(tmpRoot, "untouched-target.txt");
|
|
1008
|
+
fs.writeFileSync(targetPath, "do-not-touch");
|
|
1009
|
+
const knownTargetMs = 1_500_000_000_000;
|
|
1010
|
+
fs.utimesSync(targetPath, new Date(knownTargetMs), new Date(knownTargetMs));
|
|
1011
|
+
|
|
1012
|
+
const linkPath = path.join(tmpRoot, "the-link");
|
|
1013
|
+
nextGetObjectResponse = {
|
|
1014
|
+
Body: (async function* () {
|
|
1015
|
+
yield Buffer.from(SYMLINK_BODY_PREFIX + "untouched-target.txt");
|
|
1016
|
+
})(),
|
|
1017
|
+
Metadata: {
|
|
1018
|
+
"hq-symlink-target": SYMLINK_MARKER_META_VALUE,
|
|
1019
|
+
// Even if a misbehaving uploader stamped hq-mtime on a symlink
|
|
1020
|
+
// record, the downloader's symlink branch returns BEFORE the
|
|
1021
|
+
// utimes block, so the target's mtime stays at knownTargetMs.
|
|
1022
|
+
[FILE_MTIME_META_KEY]: "1_900_000_000_000",
|
|
1023
|
+
},
|
|
1024
|
+
};
|
|
1025
|
+
|
|
1026
|
+
await downloadFile(makeCtx(), "the-link", linkPath);
|
|
1027
|
+
|
|
1028
|
+
expect(fs.lstatSync(linkPath).isSymbolicLink()).toBe(true);
|
|
1029
|
+
expect(fs.readlinkSync(linkPath)).toBe("untouched-target.txt");
|
|
1030
|
+
// Target's mtime is unchanged — utimes block was skipped.
|
|
1031
|
+
const targetStat = fs.statSync(targetPath);
|
|
1032
|
+
expect(Math.abs(targetStat.mtimeMs - knownTargetMs)).toBeLessThanOrEqual(50);
|
|
1033
|
+
});
|
|
1034
|
+
|
|
1035
|
+
it("round-trips mtime across upload + download (5.37.0 user-visible promise)", async () => {
|
|
1036
|
+
// The whole point of the feature: a file's mtime should follow it
|
|
1037
|
+
// across the wire. Push captures source mtime → metadata; pull applies
|
|
1038
|
+
// metadata → local mtime. This test wires upload's PutObject capture
|
|
1039
|
+
// into download's GetObject response so the contract round-trips end-
|
|
1040
|
+
// to-end through s3.ts without bringing in journal/CLI state.
|
|
1041
|
+
const sourceMs = 1_650_000_000_000; // 2022-04-15T03:33:20Z
|
|
1042
|
+
const sourcePath = path.join(tmpRoot, "round-trip-src.bin");
|
|
1043
|
+
fs.writeFileSync(sourcePath, "round-trip-bytes");
|
|
1044
|
+
fs.utimesSync(sourcePath, new Date(sourceMs), new Date(sourceMs));
|
|
1045
|
+
|
|
1046
|
+
sentCommands.length = 0;
|
|
1047
|
+
await uploadFile(makeCtx(), sourcePath, "round-trip.bin");
|
|
1048
|
+
const put = sentCommands.find((c) => c.name === "PutObjectCommand");
|
|
1049
|
+
const stampedMeta = put!.input.Metadata as Record<string, string>;
|
|
1050
|
+
const stampedBody = put!.input.Body as Buffer;
|
|
1051
|
+
|
|
1052
|
+
// Wipe the local copy and pull via download — simulating a different
|
|
1053
|
+
// machine receiving the file.
|
|
1054
|
+
fs.unlinkSync(sourcePath);
|
|
1055
|
+
nextGetObjectResponse = {
|
|
1056
|
+
Body: (async function* () {
|
|
1057
|
+
yield new Uint8Array(stampedBody);
|
|
1058
|
+
})(),
|
|
1059
|
+
Metadata: stampedMeta,
|
|
1060
|
+
};
|
|
1061
|
+
await downloadFile(makeCtx(), "round-trip.bin", sourcePath);
|
|
1062
|
+
|
|
1063
|
+
const stat = fs.statSync(sourcePath);
|
|
1064
|
+
// Within 100ms — covers FS rounding on both ends of the round-trip.
|
|
1065
|
+
expect(Math.abs(stat.mtimeMs - sourceMs)).toBeLessThanOrEqual(100);
|
|
1066
|
+
expect(fs.readFileSync(sourcePath, "utf-8")).toBe("round-trip-bytes");
|
|
1067
|
+
});
|
|
1068
|
+
|
|
773
1069
|
it("returns the object's user-metadata (including created-by) for a regular file", async () => {
|
|
774
1070
|
nextGetObjectResponse = {
|
|
775
1071
|
Body: (async function* () {
|
package/src/s3.ts
CHANGED
|
@@ -182,6 +182,62 @@ export const SYMLINK_BODY_PREFIX = "hq-symlink:";
|
|
|
182
182
|
*/
|
|
183
183
|
export const FILE_MODE_META_KEY = "hq-mode";
|
|
184
184
|
|
|
185
|
+
/**
|
|
186
|
+
* S3 user-metadata key carrying the source-side file modification time
|
|
187
|
+
* (mtimeMs) as an integer-millisecond epoch string ("1700000000000"). On
|
|
188
|
+
* download, downloadFile parses this with a strict-numeric regex BEFORE
|
|
189
|
+
* parseInt, then applies it via `fs.utimesSync(localPath, mtimeDate,
|
|
190
|
+
* mtimeDate)` after the byte write.
|
|
191
|
+
*
|
|
192
|
+
* 5.37.0 symmetric to the 5.34.0 Bug #5 mode preservation: a file's
|
|
193
|
+
* modification time should follow it across machines instead of resetting
|
|
194
|
+
* to "the time of sync." Without this metadata, every receiver's mtime is
|
|
195
|
+
* wall-clock-now at write-time — making "newer than" comparisons,
|
|
196
|
+
* mtime-keyed caches, and reproducible builds break across sync.
|
|
197
|
+
*
|
|
198
|
+
* Symlinks: skipped at upload time (symlink mtime is OS-controlled and
|
|
199
|
+
* `lstat` on a symlink already returns the symlink's own times — but we
|
|
200
|
+
* don't stamp them because the symlink record wire body is `hq-symlink:`
|
|
201
|
+
* + target string, not real file content, so its mtime isn't user-
|
|
202
|
+
* meaningful) and skipped on download (`fs.utimesSync` follows symlinks
|
|
203
|
+
* and would mutate the target's mtime instead; `lutimesSync` is not in
|
|
204
|
+
* stable Node).
|
|
205
|
+
*
|
|
206
|
+
* Composition with 5.36.0 lstat fast-path: the journal stamp is captured
|
|
207
|
+
* AFTER utimesSync runs (the share/sync call sites lstat AFTER
|
|
208
|
+
* downloadFile returns), so the journal's mtimeMs matches the post-utimes
|
|
209
|
+
* lstat. The next sync's fast-path correctly skips re-hashing.
|
|
210
|
+
*
|
|
211
|
+
* Clock skew: a peer with a wrong clock pushes file with mtimeMs=<wrong>;
|
|
212
|
+
* receivers apply <wrong>. This is the same trade git's "file from the
|
|
213
|
+
* future" warning makes — silent in our case, deliberately. Clock skew
|
|
214
|
+
* is the user's problem, not the sync engine's.
|
|
215
|
+
*
|
|
216
|
+
* Back-compat: legacy uploads (pre-5.37.0) have no `hq-mtime` header —
|
|
217
|
+
* the receiver leaves the on-disk mtime at write-time, matching pre-
|
|
218
|
+
* 5.37.0 behavior. Forward-compat: pre-5.37.0 pullers ignore `hq-mtime`
|
|
219
|
+
* and keep their current "mtime = write-time" behavior. Both work; only
|
|
220
|
+
* the receiver upgrade unlocks the feature.
|
|
221
|
+
*/
|
|
222
|
+
export const FILE_MTIME_META_KEY = "hq-mtime";
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* S3 user-metadata key carrying the source-side file birthtime (birthtimeMs)
|
|
226
|
+
* as an integer-millisecond epoch string. Stamped on upload ONLY when
|
|
227
|
+
* `birthtimeMs > 0 && birthtimeMs !== mtimeMs` — many filesystems (Linux
|
|
228
|
+
* ext4 historically, tmpfs, some FUSE mounts) return 0 (unsupported) or
|
|
229
|
+
* the same value as mtime (no separate creation time tracking). The
|
|
230
|
+
* filter keeps the metadata header free of noise on those platforms.
|
|
231
|
+
*
|
|
232
|
+
* Pull: NO-OP for now. Node has no API to set birthtime on POSIX as of
|
|
233
|
+
* v24 (no `lbirthtime`, no `birthtimeSync`). The push side stamps it
|
|
234
|
+
* anyway so a future receiver upgrade — once Node lands the API — can
|
|
235
|
+
* apply it without a server-side data migration.
|
|
236
|
+
*
|
|
237
|
+
* Symlinks: skipped on both sides for the same reasons as `hq-mtime`.
|
|
238
|
+
*/
|
|
239
|
+
export const FILE_BTIME_META_KEY = "hq-btime";
|
|
240
|
+
|
|
185
241
|
/**
|
|
186
242
|
* Encode/decode the symlink wire body. Kept as exported helpers so the
|
|
187
243
|
* format is centrally defined and tests can probe both sides without
|
|
@@ -204,14 +260,61 @@ export async function uploadFile(
|
|
|
204
260
|
// FILE_MODE_META_KEY doc. Best-effort: lstat failure (raced rm, EPERM)
|
|
205
261
|
// falls through to "no mode header" and the receiver keeps its umask
|
|
206
262
|
// default — same as the legacy back-compat path.
|
|
263
|
+
//
|
|
264
|
+
// 5.37.0: same lstat call also yields mtimeMs + birthtimeMs for the
|
|
265
|
+
// hq-mtime / hq-btime metadata headers. Single lstat keeps the syscall
|
|
266
|
+
// budget identical to 5.36.0; the additional metadata fields are pure
|
|
267
|
+
// in-memory work on the lstat result. Symlinks skip ALL THREE stamps
|
|
268
|
+
// — symlink mode is OS-controlled, symlink mtime isn't user-meaningful
|
|
269
|
+
// because the wire body is `hq-symlink:` + target (not real file
|
|
270
|
+
// content), and fs.utimesSync follows links so applying it on receive
|
|
271
|
+
// would mutate the target's mtime instead of the link's.
|
|
207
272
|
let modeOctal: string | undefined;
|
|
273
|
+
let mtimeMsStamp: string | undefined;
|
|
274
|
+
let btimeMsStamp: string | undefined;
|
|
208
275
|
try {
|
|
209
276
|
const lstat = fs.lstatSync(localPath);
|
|
210
277
|
if (!lstat.isSymbolicLink()) {
|
|
211
278
|
modeOctal = (lstat.mode & 0o777).toString(8);
|
|
279
|
+
// Math.floor truncates the sub-millisecond fractional component
|
|
280
|
+
// some filesystems report (APFS returns full ms+fraction; ext4
|
|
281
|
+
// is integer-ms). String(int) on the read side matches the
|
|
282
|
+
// strict-numeric regex `^-?[0-9]{1,16}$` — optional leading `-`,
|
|
283
|
+
// no leading zeros, no decimals, no exponents.
|
|
284
|
+
//
|
|
285
|
+
// Codex PR #27 P2: accept the full finite range, including 0
|
|
286
|
+
// (Unix epoch) and negatives (pre-epoch / reproducible-build
|
|
287
|
+
// clamping). Earlier `> 0` filter silently dropped legitimate
|
|
288
|
+
// timestamps and broke the round-trip guarantee for that subset.
|
|
289
|
+
const mtimeFloor = Math.floor(lstat.mtimeMs);
|
|
290
|
+
if (Number.isFinite(lstat.mtimeMs)) {
|
|
291
|
+
mtimeMsStamp = String(mtimeFloor);
|
|
292
|
+
}
|
|
293
|
+
// birthtimeMs filter: only stamp when the filesystem actually
|
|
294
|
+
// tracks a separate creation time. ext4 historically returns 0
|
|
295
|
+
// (unsupported) or equals mtimeMs (no distinct tracking); tmpfs
|
|
296
|
+
// and some FUSE mounts behave similarly. Filtering at the source
|
|
297
|
+
// keeps the metadata header free of noise — the receiver can
|
|
298
|
+
// assume hq-btime, if present, carries real signal.
|
|
299
|
+
//
|
|
300
|
+
// Compare the floored values (not raw lstat.birthtimeMs vs
|
|
301
|
+
// lstat.mtimeMs) because APFS exposes sub-millisecond fractions —
|
|
302
|
+
// two timestamps representing the "same moment" for sync purposes
|
|
303
|
+
// can differ by < 1 ms and pass a strict `!==` check while serializing
|
|
304
|
+
// to the same integer-ms string. Comparing floor-to-floor matches
|
|
305
|
+
// what we actually emit on the wire.
|
|
306
|
+
const btimeFloor = Math.floor(lstat.birthtimeMs);
|
|
307
|
+
if (
|
|
308
|
+
Number.isFinite(lstat.birthtimeMs) &&
|
|
309
|
+
btimeFloor > 0 &&
|
|
310
|
+
btimeFloor !== mtimeFloor
|
|
311
|
+
) {
|
|
312
|
+
btimeMsStamp = String(btimeFloor);
|
|
313
|
+
}
|
|
212
314
|
}
|
|
213
315
|
} catch {
|
|
214
|
-
// Leave
|
|
316
|
+
// Leave stamps undefined; receiver applies its umask default and
|
|
317
|
+
// leaves mtime at write-time (the legacy back-compat path).
|
|
215
318
|
}
|
|
216
319
|
|
|
217
320
|
// Preserve the original `created-at` across re-uploads when the object
|
|
@@ -237,6 +340,8 @@ export async function uploadFile(
|
|
|
237
340
|
const Metadata: Record<string, string> = {
|
|
238
341
|
...(author ? buildAuthorMetadata(author, createdAt) : {}),
|
|
239
342
|
...(modeOctal ? { [FILE_MODE_META_KEY]: modeOctal } : {}),
|
|
343
|
+
...(mtimeMsStamp ? { [FILE_MTIME_META_KEY]: mtimeMsStamp } : {}),
|
|
344
|
+
...(btimeMsStamp ? { [FILE_BTIME_META_KEY]: btimeMsStamp } : {}),
|
|
240
345
|
};
|
|
241
346
|
|
|
242
347
|
const response = await client.send(
|
|
@@ -473,6 +578,55 @@ export async function downloadFile(
|
|
|
473
578
|
}
|
|
474
579
|
}
|
|
475
580
|
|
|
581
|
+
// 5.37.0 — apply source-side mtime after the byte write (and after the
|
|
582
|
+
// chmod above; ordering between chmod and utimes doesn't matter, but
|
|
583
|
+
// both must run AFTER writeFileSync because writeFileSync resets mtime
|
|
584
|
+
// to wall-clock-now). See FILE_MTIME_META_KEY for the metadata contract.
|
|
585
|
+
//
|
|
586
|
+
// Strict-numeric regex BEFORE parseInt — same Codex P2 lesson as hq-mode.
|
|
587
|
+
// `^-?[0-9]{1,16}$` rejects partial-prefix garbage ("175junk" → 175),
|
|
588
|
+
// empty, double-signed, decimals, whitespace, and oversized strings. A
|
|
589
|
+
// single optional leading `-` is allowed so pre-epoch / reproducible-build
|
|
590
|
+
// timestamps round-trip (Codex PR #27 P2 — `mtimeMs === 0` and negative
|
|
591
|
+
// epoch values are legitimate). 16 digits comfortably covers any plausible
|
|
592
|
+
// epoch-ms value (year ~5138 is 16 digits; we'll cross that bridge later).
|
|
593
|
+
//
|
|
594
|
+
// fs.utimesSync FOLLOWS symlinks — that's fine here because we're on
|
|
595
|
+
// the regular-file branch (the symlink branch above already returned
|
|
596
|
+
// before we reach this code).
|
|
597
|
+
//
|
|
598
|
+
// Composition with the 5.36.0 lstat fast-path: the journal stamp at the
|
|
599
|
+
// share/sync call sites runs AFTER downloadFile returns, so the lstat
|
|
600
|
+
// it captures sees the post-utimes mtime. Verified in cli/sync.ts
|
|
601
|
+
// (downloadFile → lstatSync → updateEntry) and cli/share.ts (pull path
|
|
602
|
+
// similarly lstats after downloadFile). If a future caller stamps the
|
|
603
|
+
// journal BEFORE downloadFile completes, the fast-path will stale and
|
|
604
|
+
// re-hash every sync forever — keep the call-site invariant intact.
|
|
605
|
+
const mtimeRaw = response.Metadata?.[FILE_MTIME_META_KEY];
|
|
606
|
+
if (typeof mtimeRaw === "string" && /^-?[0-9]{1,16}$/.test(mtimeRaw)) {
|
|
607
|
+
const mtimeMs = parseInt(mtimeRaw, 10);
|
|
608
|
+
if (Number.isFinite(mtimeMs)) {
|
|
609
|
+
try {
|
|
610
|
+
// utimesSync accepts seconds OR Date; use Date(ms) for precision.
|
|
611
|
+
// atime = mtime is fine — many filesystems are mounted noatime
|
|
612
|
+
// and distinguishing "access" vs "modification" time doesn't
|
|
613
|
+
// matter for sync semantics. Setting both keeps the on-disk
|
|
614
|
+
// state deterministic across receivers.
|
|
615
|
+
const mtimeDate = new Date(mtimeMs);
|
|
616
|
+
fs.utimesSync(localPath, mtimeDate, mtimeDate);
|
|
617
|
+
} catch {
|
|
618
|
+
// EPERM / read-only FS / file just unlinked → non-fatal. The
|
|
619
|
+
// file is materialized at write-time mtime; the source-of-truth
|
|
620
|
+
// can re-sync next pass.
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// TODO: stamp hq-btime once Node lands lbirthtime (or birthtimeSync).
|
|
626
|
+
// The push side already emits hq-btime when the source FS tracks a
|
|
627
|
+
// distinct creation time, so a future receiver upgrade picks it up
|
|
628
|
+
// automatically without a server-side data migration.
|
|
629
|
+
|
|
476
630
|
return { metadata: response.Metadata };
|
|
477
631
|
}
|
|
478
632
|
|