@indigoai-us/hq-cloud 5.17.0 → 5.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +19 -0
- package/.github/workflows/publish.yml +53 -0
- package/dist/cli/share.d.ts +7 -5
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +189 -18
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +292 -2
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +98 -17
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +302 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal.d.ts +26 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +31 -0
- package/dist/journal.js.map +1 -1
- package/dist/s3.d.ts +91 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +245 -0
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +347 -1
- package/dist/s3.test.js.map +1 -1
- package/package.json +1 -1
- package/src/cli/share.test.ts +365 -2
- package/src/cli/share.ts +241 -28
- package/src/cli/sync.test.ts +345 -0
- package/src/cli/sync.ts +133 -24
- package/src/journal.ts +33 -0
- package/src/s3.test.ts +415 -1
- package/src/s3.ts +271 -0
- package/tsconfig.json +12 -1
package/src/cli/sync.test.ts
CHANGED
|
@@ -740,4 +740,349 @@ describe("sync", () => {
|
|
|
740
740
|
filesToSkip: 0,
|
|
741
741
|
});
|
|
742
742
|
});
|
|
743
|
+
|
|
744
|
+
it("stamps the journal with sha256(target) + size 0 when the downloaded entry is a symlink", async () => {
|
|
745
|
+
// Pre-fix, the post-download journal stamp called hashFile (which
|
|
746
|
+
// follows the link and reads the target file's bytes) and statSync
|
|
747
|
+
// (which follows the link and reports the target's size). The next
|
|
748
|
+
// push's skipUnchanged check would then compare its own
|
|
749
|
+
// sha256(target string) against the journal's "target file bytes"
|
|
750
|
+
// hash, never match, and re-upload every symlink on every tick. The
|
|
751
|
+
// fix: when localPath is a symlink after downloadFile, stamp the
|
|
752
|
+
// hash of readlink(localPath) and size 0 — exactly the values the
|
|
753
|
+
// push side would produce next.
|
|
754
|
+
const linkKey = "policies/personal-link.md";
|
|
755
|
+
const linkTarget = "../../personal/policies/real.md";
|
|
756
|
+
|
|
757
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
758
|
+
{ key: linkKey, size: 0, lastModified: new Date(), etag: '"link-etag"' },
|
|
759
|
+
]);
|
|
760
|
+
vi.mocked(s3Module.downloadFile).mockImplementationOnce(
|
|
761
|
+
async (_ctx: unknown, _key: string, localPath: string) => {
|
|
762
|
+
const dir = path.dirname(localPath);
|
|
763
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
764
|
+
// Mirror real downloadFile's symlink branch.
|
|
765
|
+
fs.symlinkSync(linkTarget, localPath);
|
|
766
|
+
},
|
|
767
|
+
);
|
|
768
|
+
|
|
769
|
+
const result = await sync({
|
|
770
|
+
company: "acme",
|
|
771
|
+
vaultConfig: mockConfig,
|
|
772
|
+
hqRoot: tmpDir,
|
|
773
|
+
});
|
|
774
|
+
expect(result.filesDownloaded).toBe(1);
|
|
775
|
+
|
|
776
|
+
// Recompute the expected hash from the target string — same helper
|
|
777
|
+
// the push side uses in computePushPlan for kind: "symlink", so
|
|
778
|
+
// skip-unchanged works on the next tick.
|
|
779
|
+
const journalMod = await import("../journal.js");
|
|
780
|
+
const expectedHash = journalMod.hashSymlinkTarget(linkTarget);
|
|
781
|
+
|
|
782
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
783
|
+
expect(journal.files[linkKey]).toMatchObject({
|
|
784
|
+
hash: expectedHash,
|
|
785
|
+
size: 0,
|
|
786
|
+
direction: "down",
|
|
787
|
+
});
|
|
788
|
+
});
|
|
789
|
+
|
|
790
|
+
it("does not crash when local is a real directory at a remote-key path (EISDIR guard)", async () => {
|
|
791
|
+
// Codex round-9 P2 follow-up: pre-fix, the pull planner's
|
|
792
|
+
// post-lstat branch always called hashFile (readFileSync) on a
|
|
793
|
+
// localPath whose lstat succeeded but wasn't a symlink. For a
|
|
794
|
+
// local *directory* at a remote-key path (e.g. user has
|
|
795
|
+
// `companies/acme/knowledge/` as a real dir, cloud has a single
|
|
796
|
+
// object at the same key), readFileSync threw EISDIR and the
|
|
797
|
+
// entire sync run aborted during planning. The fix surfaces it
|
|
798
|
+
// as a non-fatal skip-local-only with a warning so the operator
|
|
799
|
+
// can manually reconcile rather than crash the whole pull.
|
|
800
|
+
const localDirKey = "knowledge";
|
|
801
|
+
const localDir = path.join(tmpDir, "companies", "acme", localDirKey);
|
|
802
|
+
fs.mkdirSync(localDir, { recursive: true });
|
|
803
|
+
fs.writeFileSync(path.join(localDir, "inside.md"), "local-only content");
|
|
804
|
+
|
|
805
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
806
|
+
{
|
|
807
|
+
key: localDirKey,
|
|
808
|
+
size: 100,
|
|
809
|
+
lastModified: new Date(),
|
|
810
|
+
etag: '"some-etag"',
|
|
811
|
+
},
|
|
812
|
+
]);
|
|
813
|
+
const warnSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
|
814
|
+
|
|
815
|
+
let crashed = false;
|
|
816
|
+
let result;
|
|
817
|
+
try {
|
|
818
|
+
result = await sync({
|
|
819
|
+
company: "acme",
|
|
820
|
+
vaultConfig: mockConfig,
|
|
821
|
+
hqRoot: tmpDir,
|
|
822
|
+
});
|
|
823
|
+
} catch {
|
|
824
|
+
crashed = true;
|
|
825
|
+
}
|
|
826
|
+
warnSpy.mockRestore();
|
|
827
|
+
|
|
828
|
+
expect(crashed).toBe(false);
|
|
829
|
+
expect(result).toBeDefined();
|
|
830
|
+
// Skipped, not downloaded — destructive replacement of a local
|
|
831
|
+
// directory is a manual-reconciliation decision, not an automatic
|
|
832
|
+
// sync action.
|
|
833
|
+
expect(result!.filesDownloaded).toBe(0);
|
|
834
|
+
expect(result!.filesSkipped).toBe(1);
|
|
835
|
+
// Local directory still intact.
|
|
836
|
+
expect(fs.existsSync(path.join(localDir, "inside.md"))).toBe(true);
|
|
837
|
+
});
|
|
838
|
+
|
|
839
|
+
it("pulls a remote symlink record under an .hqinclude dir-only allowlist pattern", async () => {
|
|
840
|
+
// Codex round-7 P1 follow-up: pre-fix, computePullPlan called
|
|
841
|
+
// shouldSync(localPath) with the default isDir=false. LIST gives
|
|
842
|
+
// no file-vs-dir signal, so a remote symlink record at a path
|
|
843
|
+
// covered only by a dir-only .hqinclude pattern (e.g.
|
|
844
|
+
// `companies/*/knowledge/`) would be classified skip-ignored —
|
|
845
|
+
// the symlink record uploaded by the push-side fix would never
|
|
846
|
+
// propagate to peers. Symmetric to the walkDir/collectFiles
|
|
847
|
+
// dual-hint fix on the push side.
|
|
848
|
+
const linkKey = "knowledge";
|
|
849
|
+
const linkTarget = "../../repos/private/knowledge-acme";
|
|
850
|
+
// Allowlist with ONLY a dir-only pattern. The pre-fix bug was
|
|
851
|
+
// that the slashless probe didn't match this rule, so a bare
|
|
852
|
+
// `knowledge` alternative would mask the regression — keep the
|
|
853
|
+
// pattern strictly dir-only so the dual-hint probe is what
|
|
854
|
+
// actually has to make the test pass.
|
|
855
|
+
fs.writeFileSync(path.join(tmpDir, ".hqinclude"), "knowledge/\n");
|
|
856
|
+
|
|
857
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
858
|
+
{ key: linkKey, size: 5, lastModified: new Date(), etag: '"link-etag-3"' },
|
|
859
|
+
]);
|
|
860
|
+
let downloadCalls = 0;
|
|
861
|
+
vi.mocked(s3Module.downloadFile).mockImplementationOnce(
|
|
862
|
+
async (_ctx: unknown, _key: string, localPath: string) => {
|
|
863
|
+
downloadCalls++;
|
|
864
|
+
const dir = path.dirname(localPath);
|
|
865
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
866
|
+
fs.symlinkSync(linkTarget, localPath);
|
|
867
|
+
},
|
|
868
|
+
);
|
|
869
|
+
|
|
870
|
+
const result = await sync({
|
|
871
|
+
company: "acme",
|
|
872
|
+
vaultConfig: mockConfig,
|
|
873
|
+
hqRoot: tmpDir,
|
|
874
|
+
});
|
|
875
|
+
|
|
876
|
+
// Pre-fix: filesDownloaded=0, filesSkipped=1 (skip-ignored).
|
|
877
|
+
// Post-fix: filesDownloaded=1.
|
|
878
|
+
expect(downloadCalls).toBe(1);
|
|
879
|
+
expect(result.filesDownloaded).toBe(1);
|
|
880
|
+
expect(
|
|
881
|
+
fs
|
|
882
|
+
.lstatSync(path.join(tmpDir, "companies", "acme", linkKey))
|
|
883
|
+
.isSymbolicLink(),
|
|
884
|
+
).toBe(true);
|
|
885
|
+
});
|
|
886
|
+
|
|
887
|
+
it("does not flag a pristine local symlink as locally-changed in the pull planner", async () => {
|
|
888
|
+
// Codex P2 follow-up: pre-fix, computePullPlan called hashFile on
|
|
889
|
+
// localPath (which follows symlinks → hashes the target file's
|
|
890
|
+
// bytes) and compared that against the journal's
|
|
891
|
+
// sha256(target string). The hashes never matched, so a pristine
|
|
892
|
+
// symlink was always classified localChanged. Combined with a
|
|
893
|
+
// remote-side change from another machine, that mis-classified as a
|
|
894
|
+
// conflict and aborted overwrites; without remote change, it was
|
|
895
|
+
// only logged as skip-local-only. Either way the planner thought
|
|
896
|
+
// the local link was "edited" when it wasn't.
|
|
897
|
+
const linkKey = "policies/personal-link.md";
|
|
898
|
+
const linkTarget = "../../personal/policies/real.md";
|
|
899
|
+
// listRemoteFiles surfaces the raw S3 etag with literal quotes;
|
|
900
|
+
// the journal stamps the normalized (unquoted) form. Match the
|
|
901
|
+
// journal convention here so hasRemoteChanged returns false.
|
|
902
|
+
const remoteEtagWire = '"link-etag-2"';
|
|
903
|
+
const remoteEtagNormalized = "link-etag-2";
|
|
904
|
+
|
|
905
|
+
// Pre-create the link locally so the planner sees an existing
|
|
906
|
+
// symlink, and pre-stamp the journal with the SAME hash the push
|
|
907
|
+
// side would have written (hashSymlinkTarget of the target string).
|
|
908
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
909
|
+
const linkDir = path.dirname(path.join(companyRoot, linkKey));
|
|
910
|
+
fs.mkdirSync(linkDir, { recursive: true });
|
|
911
|
+
fs.symlinkSync(linkTarget, path.join(companyRoot, linkKey));
|
|
912
|
+
const journalMod = await import("../journal.js");
|
|
913
|
+
const matchingHash = journalMod.hashSymlinkTarget(linkTarget);
|
|
914
|
+
fs.writeFileSync(
|
|
915
|
+
journalPath,
|
|
916
|
+
JSON.stringify({
|
|
917
|
+
version: "1",
|
|
918
|
+
lastSync: new Date().toISOString(),
|
|
919
|
+
files: {
|
|
920
|
+
[linkKey]: {
|
|
921
|
+
hash: matchingHash,
|
|
922
|
+
size: 0,
|
|
923
|
+
syncedAt: new Date().toISOString(),
|
|
924
|
+
direction: "down",
|
|
925
|
+
remoteEtag: remoteEtagNormalized,
|
|
926
|
+
},
|
|
927
|
+
},
|
|
928
|
+
}),
|
|
929
|
+
);
|
|
930
|
+
|
|
931
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValue([
|
|
932
|
+
{ key: linkKey, size: 0, lastModified: new Date(), etag: remoteEtagWire },
|
|
933
|
+
]);
|
|
934
|
+
// downloadFile must NOT be called: the planner should skip
|
|
935
|
+
// (unchanged on both sides). Use a sentinel that throws if invoked.
|
|
936
|
+
vi.mocked(s3Module.downloadFile).mockImplementation(async () => {
|
|
937
|
+
throw new Error("download must not be called for unchanged symlink");
|
|
938
|
+
});
|
|
939
|
+
|
|
940
|
+
const result = await sync({
|
|
941
|
+
company: "acme",
|
|
942
|
+
vaultConfig: mockConfig,
|
|
943
|
+
hqRoot: tmpDir,
|
|
944
|
+
});
|
|
945
|
+
|
|
946
|
+
expect(result.filesDownloaded).toBe(0);
|
|
947
|
+
expect(result.filesSkipped).toBe(1);
|
|
948
|
+
expect(result.conflicts).toBe(0);
|
|
949
|
+
});
|
|
950
|
+
|
|
951
|
+
it("classifies a dangling-symlink + remote-change as a conflict without crashing on statSync", async () => {
|
|
952
|
+
// Codex round-11 P2 follow-up: pre-fix, the conflict executor
|
|
953
|
+
// built the resolveConflict prompt with `fs.statSync(localPath).
|
|
954
|
+
// mtime`, which follows symlinks. For a dangling symlink (target
|
|
955
|
+
// missing locally) that ALSO had a target rewrite (localChanged)
|
|
956
|
+
// and a remote change (remoteChanged), the planner classified it
|
|
957
|
+
// as a conflict, then statSync threw ENOENT trying to follow the
|
|
958
|
+
// dangling link to read mtime — aborting the entire pull before
|
|
959
|
+
// resolveConflict could run. Fix: carry the planner's lstat
|
|
960
|
+
// mtime forward through PullPlanItem.conflict.localMtime so the
|
|
961
|
+
// executor never re-stats the path.
|
|
962
|
+
const linkKey = "dangling-link.md";
|
|
963
|
+
// Old target — what the journal-recorded hash will reflect.
|
|
964
|
+
const oldTarget = "old-target.md";
|
|
965
|
+
// New (current) target — points to a path that doesn't exist
|
|
966
|
+
// locally → the link is dangling. The hash differs from the
|
|
967
|
+
// journal so localChanged = true.
|
|
968
|
+
const newTarget = "missing-target.md";
|
|
969
|
+
|
|
970
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
971
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
972
|
+
fs.symlinkSync(newTarget, path.join(companyRoot, linkKey));
|
|
973
|
+
|
|
974
|
+
// Pre-stamp the journal with the OLD target's symlink hash, plus a
|
|
975
|
+
// remoteEtag that won't match the LIST etag we mock (forcing
|
|
976
|
+
// remoteChanged = true on top of localChanged = true).
|
|
977
|
+
const journalMod = await import("../journal.js");
|
|
978
|
+
const oldHash = journalMod.hashSymlinkTarget(oldTarget);
|
|
979
|
+
fs.writeFileSync(
|
|
980
|
+
journalPath,
|
|
981
|
+
JSON.stringify({
|
|
982
|
+
version: "1",
|
|
983
|
+
lastSync: new Date().toISOString(),
|
|
984
|
+
files: {
|
|
985
|
+
[linkKey]: {
|
|
986
|
+
hash: oldHash,
|
|
987
|
+
size: 0,
|
|
988
|
+
syncedAt: new Date().toISOString(),
|
|
989
|
+
direction: "down",
|
|
990
|
+
remoteEtag: "old-remote-etag",
|
|
991
|
+
},
|
|
992
|
+
},
|
|
993
|
+
}),
|
|
994
|
+
);
|
|
995
|
+
|
|
996
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
997
|
+
{
|
|
998
|
+
key: linkKey,
|
|
999
|
+
size: 30,
|
|
1000
|
+
lastModified: new Date(),
|
|
1001
|
+
etag: '"new-remote-etag"',
|
|
1002
|
+
},
|
|
1003
|
+
]);
|
|
1004
|
+
|
|
1005
|
+
// Auto-resolve the conflict by aborting. What we're really testing
|
|
1006
|
+
// is that the executor *reaches* resolveConflict at all — pre-fix
|
|
1007
|
+
// it crashed at fs.statSync before getting there.
|
|
1008
|
+
const result = await sync({
|
|
1009
|
+
company: "acme",
|
|
1010
|
+
vaultConfig: mockConfig,
|
|
1011
|
+
hqRoot: tmpDir,
|
|
1012
|
+
onConflict: "abort",
|
|
1013
|
+
});
|
|
1014
|
+
|
|
1015
|
+
expect(result.aborted).toBe(true);
|
|
1016
|
+
expect(result.conflicts).toBe(1);
|
|
1017
|
+
expect(result.conflictPaths).toContain(linkKey);
|
|
1018
|
+
});
|
|
1019
|
+
|
|
1020
|
+
it("conflict resolution 'keep' on a dangling symlink stamps the journal so it doesn't re-fire next sync", async () => {
|
|
1021
|
+
// Codex round-12 P2 follow-up: pre-fix, the keep/skip branch of
|
|
1022
|
+
// the conflict executor called `fs.statSync(localPath).size` to
|
|
1023
|
+
// populate the journal stamp. For a dangling symlink, statSync
|
|
1024
|
+
// followed the link and threw ENOENT, the catch swallowed it,
|
|
1025
|
+
// and the journal was never updated — so the next sync saw the
|
|
1026
|
+
// same localChanged && remoteChanged combination and prompted
|
|
1027
|
+
// for the SAME conflict forever, ignoring the user's keep
|
|
1028
|
+
// choice. Fix: carry localSize (symlink-aware: 0 for symlinks)
|
|
1029
|
+
// through the conflict item so the executor never re-stats.
|
|
1030
|
+
const linkKey = "dangling-keep.md";
|
|
1031
|
+
const oldTarget = "old-target.md";
|
|
1032
|
+
const newTarget = "missing-target.md";
|
|
1033
|
+
const newRemoteEtagWire = '"new-remote-etag-keep"';
|
|
1034
|
+
const newRemoteEtagNormalized = "new-remote-etag-keep";
|
|
1035
|
+
|
|
1036
|
+
const companyRoot = path.join(tmpDir, "companies", "acme");
|
|
1037
|
+
fs.mkdirSync(companyRoot, { recursive: true });
|
|
1038
|
+
fs.symlinkSync(newTarget, path.join(companyRoot, linkKey));
|
|
1039
|
+
|
|
1040
|
+
const journalMod = await import("../journal.js");
|
|
1041
|
+
const oldHash = journalMod.hashSymlinkTarget(oldTarget);
|
|
1042
|
+
const expectedNewHash = journalMod.hashSymlinkTarget(newTarget);
|
|
1043
|
+
fs.writeFileSync(
|
|
1044
|
+
journalPath,
|
|
1045
|
+
JSON.stringify({
|
|
1046
|
+
version: "1",
|
|
1047
|
+
lastSync: new Date().toISOString(),
|
|
1048
|
+
files: {
|
|
1049
|
+
[linkKey]: {
|
|
1050
|
+
hash: oldHash,
|
|
1051
|
+
size: 0,
|
|
1052
|
+
syncedAt: new Date().toISOString(),
|
|
1053
|
+
direction: "down",
|
|
1054
|
+
remoteEtag: "old-remote-etag-keep",
|
|
1055
|
+
},
|
|
1056
|
+
},
|
|
1057
|
+
}),
|
|
1058
|
+
);
|
|
1059
|
+
|
|
1060
|
+
vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
|
|
1061
|
+
{
|
|
1062
|
+
key: linkKey,
|
|
1063
|
+
size: 30,
|
|
1064
|
+
lastModified: new Date(),
|
|
1065
|
+
etag: newRemoteEtagWire,
|
|
1066
|
+
},
|
|
1067
|
+
]);
|
|
1068
|
+
|
|
1069
|
+
const result = await sync({
|
|
1070
|
+
company: "acme",
|
|
1071
|
+
vaultConfig: mockConfig,
|
|
1072
|
+
hqRoot: tmpDir,
|
|
1073
|
+
onConflict: "keep",
|
|
1074
|
+
});
|
|
1075
|
+
|
|
1076
|
+
expect(result.aborted).toBe(false);
|
|
1077
|
+
expect(result.conflicts).toBe(1);
|
|
1078
|
+
|
|
1079
|
+
// The journal MUST be updated to reflect the user's keep
|
|
1080
|
+
// decision. Pre-fix: catch swallowed the ENOENT, journal stayed
|
|
1081
|
+
// at oldHash + old etag, every subsequent sync re-fired.
|
|
1082
|
+
const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
|
|
1083
|
+
const entry = journal.files[linkKey];
|
|
1084
|
+
expect(entry.hash).toBe(expectedNewHash);
|
|
1085
|
+
expect(entry.size).toBe(0);
|
|
1086
|
+
expect(entry.remoteEtag).toBe(newRemoteEtagNormalized);
|
|
1087
|
+
});
|
|
743
1088
|
});
|
package/src/cli/sync.ts
CHANGED
|
@@ -11,7 +11,15 @@ import type { VaultServiceConfig, SyncJournal } from "../types.js";
|
|
|
11
11
|
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
12
12
|
import { downloadFile, listRemoteFiles, headRemoteFile } from "../s3.js";
|
|
13
13
|
import type { RemoteFile } from "../s3.js";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
readJournal,
|
|
16
|
+
writeJournal,
|
|
17
|
+
hashFile,
|
|
18
|
+
hashSymlinkTarget,
|
|
19
|
+
updateEntry,
|
|
20
|
+
getEntry,
|
|
21
|
+
normalizeEtag,
|
|
22
|
+
} from "../journal.js";
|
|
15
23
|
import { createIgnoreFilter } from "../ignore.js";
|
|
16
24
|
import { resolveConflict } from "./conflict.js";
|
|
17
25
|
import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
|
|
@@ -235,7 +243,10 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
235
243
|
path: remoteFile.key,
|
|
236
244
|
localHash: item.localHash,
|
|
237
245
|
remoteModified: remoteFile.lastModified,
|
|
238
|
-
|
|
246
|
+
// Use the lstat-mtime captured by the planner — statSync
|
|
247
|
+
// here would follow a dangling symlink and throw ENOENT,
|
|
248
|
+
// aborting the pull before resolveConflict could prompt.
|
|
249
|
+
localModified: item.localMtime,
|
|
239
250
|
direction: "pull",
|
|
240
251
|
},
|
|
241
252
|
onConflict,
|
|
@@ -311,19 +322,20 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
311
322
|
// the next sync "no change on either side" until something new
|
|
312
323
|
// diverges. Without this, both `localChanged` and `remoteChanged`
|
|
313
324
|
// stay true forever and the conflict is sticky.
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
325
|
+
// Stamp from planner-captured size (symlink-aware), NOT
|
|
326
|
+
// statSync — which would follow a dangling symlink and
|
|
327
|
+
// throw ENOENT, get swallowed, and leave the journal
|
|
328
|
+
// stale so this conflict would re-fire on every sync
|
|
329
|
+
// forever. localSize is sourced from the same lstat that
|
|
330
|
+
// computed localMtime + localHash above.
|
|
331
|
+
updateEntry(
|
|
332
|
+
journal,
|
|
333
|
+
remoteFile.key,
|
|
334
|
+
item.localHash,
|
|
335
|
+
item.localSize,
|
|
336
|
+
"down",
|
|
337
|
+
remoteFile.etag,
|
|
338
|
+
);
|
|
327
339
|
continue;
|
|
328
340
|
}
|
|
329
341
|
// "overwrite" falls through to download
|
|
@@ -333,11 +345,24 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
333
345
|
try {
|
|
334
346
|
await downloadFile(ctx, remoteFile.key, localPath);
|
|
335
347
|
|
|
336
|
-
|
|
337
|
-
|
|
348
|
+
// Symlink records materialize as real symlinks on disk. lstat
|
|
349
|
+
// (does not follow) lets us detect that case so the journal stamp
|
|
350
|
+
// mirrors what the push side would emit on the next tick:
|
|
351
|
+
// hash = sha256(readlink target string)
|
|
352
|
+
// size = 0
|
|
353
|
+
// Without this check, hashFile would follow the link and stamp the
|
|
354
|
+
// target file's contents — a value the next push would never
|
|
355
|
+
// produce — which makes skipUnchanged perpetually re-upload every
|
|
356
|
+
// symlink, defeating the point of the gate.
|
|
357
|
+
const localLstat = fs.lstatSync(localPath);
|
|
358
|
+
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
359
|
+
const hash = isLocalSymlink
|
|
360
|
+
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
361
|
+
: hashFile(localPath);
|
|
362
|
+
const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
|
|
338
363
|
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
339
364
|
// drift independently of mtime drift.
|
|
340
|
-
updateEntry(journal, remoteFile.key, hash,
|
|
365
|
+
updateEntry(journal, remoteFile.key, hash, size, "down", remoteFile.etag);
|
|
341
366
|
|
|
342
367
|
// Attach message from the prior journal entry if present (set by a
|
|
343
368
|
// previous `share` operation that included a --message).
|
|
@@ -346,12 +371,12 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
346
371
|
emit({
|
|
347
372
|
type: "progress",
|
|
348
373
|
path: remoteFile.key,
|
|
349
|
-
bytes:
|
|
374
|
+
bytes: size,
|
|
350
375
|
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
351
376
|
});
|
|
352
377
|
|
|
353
378
|
filesDownloaded++;
|
|
354
|
-
bytesDownloaded +=
|
|
379
|
+
bytesDownloaded += size;
|
|
355
380
|
} catch (err) {
|
|
356
381
|
// STS session policy may deny access to some paths — this is expected
|
|
357
382
|
// for guest members with allowedPrefixes
|
|
@@ -471,6 +496,22 @@ type PullPlanItem =
|
|
|
471
496
|
remoteFile: RemoteFile;
|
|
472
497
|
localPath: string;
|
|
473
498
|
localHash: string;
|
|
499
|
+
// Captured from the planner's lstat (does NOT follow symlinks),
|
|
500
|
+
// so a dangling symlink doesn't make the conflict executor
|
|
501
|
+
// statSync-then-ENOENT and abort the whole pull. See the
|
|
502
|
+
// executor: prior to this field, the prompt builder called
|
|
503
|
+
// fs.statSync(localPath).mtime to populate the conflict UI's
|
|
504
|
+
// "local modified at X" line, which crashed for dangling links.
|
|
505
|
+
localMtime: Date;
|
|
506
|
+
// Symlink-aware size for the post-resolution journal stamp.
|
|
507
|
+
// For a symlink, journal convention is 0 (the wire body is
|
|
508
|
+
// empty + target metadata; matches what the push side stamps).
|
|
509
|
+
// For a regular file, mirrors the on-disk byte length. The
|
|
510
|
+
// executor's keep/skip branch reads this instead of calling
|
|
511
|
+
// fs.statSync(localPath), which would follow a dangling link
|
|
512
|
+
// and silently fail to stamp the journal — leaving the
|
|
513
|
+
// conflict to re-fire forever.
|
|
514
|
+
localSize: number;
|
|
474
515
|
};
|
|
475
516
|
|
|
476
517
|
interface PullPlan {
|
|
@@ -499,7 +540,7 @@ function computePullPlan(
|
|
|
499
540
|
remoteFiles: RemoteFile[],
|
|
500
541
|
journal: SyncJournal,
|
|
501
542
|
companyRoot: string,
|
|
502
|
-
shouldSync: (filePath: string) => boolean,
|
|
543
|
+
shouldSync: (filePath: string, isDir?: boolean) => boolean,
|
|
503
544
|
personalMode: boolean,
|
|
504
545
|
): PullPlan {
|
|
505
546
|
const items: PullPlanItem[] = [];
|
|
@@ -512,16 +553,72 @@ function computePullPlan(
|
|
|
512
553
|
continue;
|
|
513
554
|
}
|
|
514
555
|
|
|
515
|
-
|
|
556
|
+
// LIST gives us no kind signal for the remote object — we don't
|
|
557
|
+
// know whether this key is a regular file or a symlink record
|
|
558
|
+
// until we either HEAD it (expensive — N extra calls per pull) or
|
|
559
|
+
// download it. So probe the ignore filter with BOTH isDir hints
|
|
560
|
+
// and include the entry if either probe passes. Without this,
|
|
561
|
+
// a symlink record at a dir-only-allowlisted path (e.g. an
|
|
562
|
+
// .hqinclude of `companies/*/knowledge/`) would be classified
|
|
563
|
+
// skip-ignored and the symlinks freshly uploaded by the
|
|
564
|
+
// push-side fix would never propagate. Mirrors the dual-hint
|
|
565
|
+
// probe in walkDir/collectFiles. Pure path lookup, no I/O.
|
|
566
|
+
if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) {
|
|
516
567
|
items.push({ action: "skip-ignored", remoteFile, localPath });
|
|
517
568
|
continue;
|
|
518
569
|
}
|
|
519
570
|
|
|
520
571
|
const journalEntry = getEntry(journal, remoteFile.key);
|
|
521
|
-
|
|
572
|
+
// lstat (not existsSync/statSync) handles three cases the legacy
|
|
573
|
+
// checks got wrong for symlinks:
|
|
574
|
+
// 1. A valid symlink at localPath: existsSync returns true and the
|
|
575
|
+
// planner falls through to hashFile, which follows the link and
|
|
576
|
+
// hashes the target file's bytes — never matching the journal's
|
|
577
|
+
// sha256(target string), so every pull would mis-classify a
|
|
578
|
+
// pristine symlink as locally changed.
|
|
579
|
+
// 2. A dangling symlink: existsSync returns false (existsSync
|
|
580
|
+
// follows links) so the planner treats the entry as missing and
|
|
581
|
+
// re-downloads, silently clobbering the user's intentional
|
|
582
|
+
// dangling link on every run.
|
|
583
|
+
// 3. A regular file: indistinguishable from current behaviour.
|
|
584
|
+
let localLstat: fs.Stats | null = null;
|
|
585
|
+
try {
|
|
586
|
+
localLstat = fs.lstatSync(localPath);
|
|
587
|
+
} catch (err: unknown) {
|
|
588
|
+
// ENOENT means truly absent; other errors propagate.
|
|
589
|
+
if (
|
|
590
|
+
err &&
|
|
591
|
+
typeof err === "object" &&
|
|
592
|
+
"code" in err &&
|
|
593
|
+
(err as { code?: string }).code !== "ENOENT"
|
|
594
|
+
) {
|
|
595
|
+
throw err;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
const localExists = localLstat !== null;
|
|
522
599
|
|
|
523
600
|
if (localExists) {
|
|
524
|
-
const
|
|
601
|
+
const isLocalSymlink = localLstat!.isSymbolicLink();
|
|
602
|
+
// Kind-mismatch guard: a remote LIST entry is always a single
|
|
603
|
+
// S3 object (regular file or symlink record), but the local
|
|
604
|
+
// path may be a real directory if the user previously had a
|
|
605
|
+
// dir at this key. Hashing a directory throws EISDIR and
|
|
606
|
+
// aborts the whole pull. We don't auto-replace because that
|
|
607
|
+
// would `rm -rf` the user's directory; surface as a non-fatal
|
|
608
|
+
// skip and warn so the operator can reconcile manually
|
|
609
|
+
// (delete the local dir, or restructure the remote).
|
|
610
|
+
if (localLstat!.isDirectory() && !isLocalSymlink) {
|
|
611
|
+
console.error(
|
|
612
|
+
` Warning: ${remoteFile.key} exists locally as a directory; ` +
|
|
613
|
+
`cloud has a single object at this key. Skipping; manual ` +
|
|
614
|
+
`reconciliation required (rm -rf the local directory to pull).`,
|
|
615
|
+
);
|
|
616
|
+
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
617
|
+
continue;
|
|
618
|
+
}
|
|
619
|
+
const localHash = isLocalSymlink
|
|
620
|
+
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
621
|
+
: hashFile(localPath);
|
|
525
622
|
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
526
623
|
const remoteChanged =
|
|
527
624
|
!!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
|
|
@@ -535,6 +632,18 @@ function computePullPlan(
|
|
|
535
632
|
remoteFile,
|
|
536
633
|
localPath,
|
|
537
634
|
localHash,
|
|
635
|
+
// localLstat is non-null inside this `if (localExists)`
|
|
636
|
+
// branch — the planner already lstat'd the path. Carry
|
|
637
|
+
// its mtime forward so the executor doesn't have to
|
|
638
|
+
// re-stat (which would follow links and crash on a
|
|
639
|
+
// dangling symlink). Same goes for size: the conflict
|
|
640
|
+
// executor's keep/skip branch needs to stamp the
|
|
641
|
+
// journal with a size, and statSync on a dangling
|
|
642
|
+
// symlink throws ENOENT (silently swallowed pre-fix),
|
|
643
|
+
// leaving the journal stale and re-firing the conflict
|
|
644
|
+
// every sync.
|
|
645
|
+
localMtime: localLstat!.mtime,
|
|
646
|
+
localSize: isLocalSymlink ? 0 : localLstat!.size,
|
|
538
647
|
});
|
|
539
648
|
continue;
|
|
540
649
|
}
|
package/src/journal.ts
CHANGED
|
@@ -74,6 +74,39 @@ export function hashFile(filePath: string): string {
|
|
|
74
74
|
return crypto.createHash("sha256").update(content).digest("hex");
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/**
|
|
78
|
+
* Marker prepended to a symlink's target string before hashing for the
|
|
79
|
+
* journal. Mirrors the wire-side `SYMLINK_BODY_PREFIX` constant in
|
|
80
|
+
* `s3.ts` — same purpose, different namespace.
|
|
81
|
+
*
|
|
82
|
+
* Without this marker, a symlink to `real.md` and a regular file whose
|
|
83
|
+
* contents are exactly the bytes `real.md` produce identical journal
|
|
84
|
+
* hashes (both `sha256("real.md")`). When `skipUnchanged` is enabled,
|
|
85
|
+
* the planner would treat a regular-file → symlink replacement as
|
|
86
|
+
* "no change" and never upload the new symlink, leaving the remote
|
|
87
|
+
* representation stale forever — the pull side would then also see no
|
|
88
|
+
* drift via ETag and never repair.
|
|
89
|
+
*
|
|
90
|
+
* Hashing `sha256(prefix + target)` makes the two representations
|
|
91
|
+
* structurally inequal in journal-hash space, so skip-unchanged can
|
|
92
|
+
* never confuse them. The hash always varies with the target string,
|
|
93
|
+
* so target rewrites still re-fire uploads as expected.
|
|
94
|
+
*/
|
|
95
|
+
export const SYMLINK_HASH_PREFIX = "hq-symlink:";
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Compute the journal hash for a symlink. Always use this helper
|
|
99
|
+
* (never inline `crypto.createHash` with the raw target) so the
|
|
100
|
+
* push side, the pull-planner, and the post-download stamp stay in
|
|
101
|
+
* lockstep on the prefixed-hash convention.
|
|
102
|
+
*/
|
|
103
|
+
export function hashSymlinkTarget(target: string): string {
|
|
104
|
+
return crypto
|
|
105
|
+
.createHash("sha256")
|
|
106
|
+
.update(SYMLINK_HASH_PREFIX + target)
|
|
107
|
+
.digest("hex");
|
|
108
|
+
}
|
|
109
|
+
|
|
77
110
|
export function updateEntry(
|
|
78
111
|
journal: SyncJournal,
|
|
79
112
|
relativePath: string,
|