@openclaw/fs-safe 0.5.6 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -0
- package/README.md +23 -10
- package/dist/advanced.d.ts +2 -1
- package/dist/advanced.d.ts.map +1 -1
- package/dist/advanced.js +1 -0
- package/dist/archive-crc32.d.ts +2 -0
- package/dist/archive-crc32.d.ts.map +1 -0
- package/dist/archive-crc32.js +14 -0
- package/dist/archive-deadline.d.ts +3 -0
- package/dist/archive-deadline.d.ts.map +1 -1
- package/dist/archive-deadline.js +44 -8
- package/dist/archive-entry.d.ts.map +1 -1
- package/dist/archive-entry.js +1 -0
- package/dist/archive-errors.d.ts +1 -0
- package/dist/archive-errors.d.ts.map +1 -1
- package/dist/archive-errors.js +3 -0
- package/dist/archive-input.d.ts.map +1 -1
- package/dist/archive-input.js +26 -16
- package/dist/archive-kind.js +2 -2
- package/dist/archive-limits.d.ts +11 -3
- package/dist/archive-limits.d.ts.map +1 -1
- package/dist/archive-limits.js +24 -0
- package/dist/archive-native.d.ts +3 -2
- package/dist/archive-native.d.ts.map +1 -1
- package/dist/archive-native.js +31 -9
- package/dist/archive-policy.d.ts +2 -0
- package/dist/archive-policy.d.ts.map +1 -1
- package/dist/archive-policy.js +9 -1
- package/dist/archive-read.d.ts.map +1 -1
- package/dist/archive-read.js +88 -41
- package/dist/archive-staging.d.ts +3 -0
- package/dist/archive-staging.d.ts.map +1 -1
- package/dist/archive-staging.js +91 -43
- package/dist/archive-tar-admission.d.ts +7 -0
- package/dist/archive-tar-admission.d.ts.map +1 -0
- package/dist/archive-tar-admission.js +43 -0
- package/dist/archive-tar-gnu.d.ts +2 -0
- package/dist/archive-tar-gnu.d.ts.map +1 -0
- package/dist/archive-tar-gnu.js +20 -0
- package/dist/archive-tar-header.d.ts +8 -0
- package/dist/archive-tar-header.d.ts.map +1 -0
- package/dist/archive-tar-header.js +47 -0
- package/dist/archive-tar-meta.d.ts +29 -1
- package/dist/archive-tar-meta.d.ts.map +1 -1
- package/dist/archive-tar-meta.js +135 -19
- package/dist/archive-tar-pax.d.ts +8 -0
- package/dist/archive-tar-pax.d.ts.map +1 -0
- package/dist/archive-tar-pax.js +100 -0
- package/dist/archive-tar-runtime.d.ts +4 -0
- package/dist/archive-tar-runtime.d.ts.map +1 -1
- package/dist/archive-tar-runtime.js +3 -0
- package/dist/archive-tar.d.ts.map +1 -1
- package/dist/archive-tar.js +8 -2
- package/dist/archive-zip-admission.d.ts +7 -0
- package/dist/archive-zip-admission.d.ts.map +1 -0
- package/dist/archive-zip-admission.js +60 -0
- package/dist/archive-zip-count.d.ts +2 -0
- package/dist/archive-zip-count.d.ts.map +1 -0
- package/dist/archive-zip-count.js +140 -0
- package/dist/archive-zip-directory.d.ts +8 -0
- package/dist/archive-zip-directory.d.ts.map +1 -0
- package/dist/archive-zip-directory.js +224 -0
- package/dist/archive-zip-integrity.d.ts.map +1 -1
- package/dist/archive-zip-integrity.js +1 -14
- package/dist/archive-zip-names.d.ts +12 -0
- package/dist/archive-zip-names.d.ts.map +1 -0
- package/dist/archive-zip-names.js +99 -0
- package/dist/archive-zip-preflight.d.ts +1 -1
- package/dist/archive-zip-preflight.d.ts.map +1 -1
- package/dist/archive-zip-preflight.js +8 -145
- package/dist/archive.d.ts.map +1 -1
- package/dist/archive.js +63 -42
- package/dist/atomic.d.ts +1 -1
- package/dist/atomic.d.ts.map +1 -1
- package/dist/bounded-read-stream.d.ts.map +1 -1
- package/dist/bounded-read-stream.js +2 -4
- package/dist/bounded-read.d.ts.map +1 -1
- package/dist/bounded-read.js +3 -10
- package/dist/byte-budget.d.ts +5 -0
- package/dist/byte-budget.d.ts.map +1 -0
- package/dist/byte-budget.js +9 -0
- package/dist/file-hash.d.ts.map +1 -1
- package/dist/file-hash.js +25 -19
- package/dist/file-lock-sync.d.ts.map +1 -1
- package/dist/file-lock-sync.js +26 -9
- package/dist/file-store-boundary.d.ts.map +1 -1
- package/dist/file-store-boundary.js +9 -5
- package/dist/file-store-limit.d.ts +2 -0
- package/dist/file-store-limit.d.ts.map +1 -0
- package/dist/file-store-limit.js +8 -0
- package/dist/file-store-sync-write.d.ts.map +1 -1
- package/dist/file-store-sync-write.js +38 -8
- package/dist/file-store.d.ts.map +1 -1
- package/dist/file-store.js +26 -26
- package/dist/json-durable-queue-directory.d.ts +2 -0
- package/dist/json-durable-queue-directory.d.ts.map +1 -0
- package/dist/json-durable-queue-directory.js +20 -0
- package/dist/json-durable-queue-ownership.d.ts +14 -0
- package/dist/json-durable-queue-ownership.d.ts.map +1 -0
- package/dist/json-durable-queue-ownership.js +168 -0
- package/dist/json-durable-queue-retirement.d.ts +9 -0
- package/dist/json-durable-queue-retirement.d.ts.map +1 -0
- package/dist/json-durable-queue-retirement.js +126 -0
- package/dist/json-durable-queue-transfer-lock.d.ts +2 -0
- package/dist/json-durable-queue-transfer-lock.d.ts.map +1 -0
- package/dist/json-durable-queue-transfer-lock.js +19 -0
- package/dist/json-durable-queue.d.ts +1 -0
- package/dist/json-durable-queue.d.ts.map +1 -1
- package/dist/json-durable-queue.js +90 -57
- package/dist/json.d.ts.map +1 -1
- package/dist/json.js +27 -8
- package/dist/local-roots.d.ts.map +1 -1
- package/dist/local-roots.js +4 -2
- package/dist/native-binding.d.ts +15 -3
- package/dist/native-binding.d.ts.map +1 -1
- package/dist/native-operations.d.ts +4 -1
- package/dist/native-operations.d.ts.map +1 -1
- package/dist/native-operations.js +22 -6
- package/dist/native-pinned-write-windows.d.ts +8 -0
- package/dist/native-pinned-write-windows.d.ts.map +1 -0
- package/dist/native-pinned-write-windows.js +92 -0
- package/dist/native-pinned-write.d.ts.map +1 -1
- package/dist/native-pinned-write.js +136 -127
- package/dist/native-staged-file.d.ts +24 -0
- package/dist/native-staged-file.d.ts.map +1 -0
- package/dist/native-staged-file.js +337 -0
- package/dist/native.d.ts.map +1 -1
- package/dist/native.js +4 -4
- package/dist/opened-realpath.d.ts +2 -0
- package/dist/opened-realpath.d.ts.map +1 -1
- package/dist/opened-realpath.js +12 -7
- package/dist/output-sibling.d.ts.map +1 -1
- package/dist/output-sibling.js +11 -110
- package/dist/output.d.ts.map +1 -1
- package/dist/output.js +4 -2
- package/dist/owner-dacl.d.ts.map +1 -1
- package/dist/owner-dacl.js +2 -1
- package/dist/permission-exec.d.ts +19 -0
- package/dist/permission-exec.d.ts.map +1 -1
- package/dist/permission-exec.js +57 -11
- package/dist/permissions-public.d.ts +1 -1
- package/dist/permissions-public.d.ts.map +1 -1
- package/dist/permissions-windows.d.ts +3 -0
- package/dist/permissions-windows.d.ts.map +1 -1
- package/dist/permissions-windows.js +16 -5
- package/dist/permissions.d.ts +5 -0
- package/dist/permissions.d.ts.map +1 -1
- package/dist/pinned-open.d.ts.map +1 -1
- package/dist/pinned-open.js +27 -48
- package/dist/pinned-write.d.ts +6 -0
- package/dist/pinned-write.d.ts.map +1 -1
- package/dist/pinned-write.js +26 -37
- package/dist/private-directory.d.ts.map +1 -1
- package/dist/private-directory.js +3 -2
- package/dist/private-temp-workspace.d.ts +3 -1
- package/dist/private-temp-workspace.d.ts.map +1 -1
- package/dist/private-temp-workspace.js +81 -56
- package/dist/publish-file.d.ts.map +1 -1
- package/dist/publish-file.js +2 -4
- package/dist/read-opened-file.d.ts.map +1 -1
- package/dist/read-opened-file.js +6 -4
- package/dist/regular-file.d.ts.map +1 -1
- package/dist/regular-file.js +136 -82
- package/dist/replace-file-copy-fallback.d.ts +3 -1
- package/dist/replace-file-copy-fallback.d.ts.map +1 -1
- package/dist/replace-file-copy-fallback.js +21 -36
- package/dist/replace-file-copy-source.d.ts +21 -0
- package/dist/replace-file-copy-source.d.ts.map +1 -0
- package/dist/replace-file-copy-source.js +112 -0
- package/dist/replace-file-descriptor.d.ts +13 -3
- package/dist/replace-file-descriptor.d.ts.map +1 -1
- package/dist/replace-file-descriptor.js +32 -7
- package/dist/replace-file-rename-policy.d.ts +7 -0
- package/dist/replace-file-rename-policy.d.ts.map +1 -0
- package/dist/replace-file-rename-policy.js +30 -0
- package/dist/replace-file-temp-owner.d.ts +46 -0
- package/dist/replace-file-temp-owner.d.ts.map +1 -0
- package/dist/replace-file-temp-owner.js +346 -0
- package/dist/replace-file.d.ts +6 -1
- package/dist/replace-file.d.ts.map +1 -1
- package/dist/replace-file.js +72 -58
- package/dist/root-impl.d.ts.map +1 -1
- package/dist/root-impl.js +118 -98
- package/dist/root-paths.d.ts +11 -14
- package/dist/root-paths.d.ts.map +1 -1
- package/dist/root-paths.js +36 -27
- package/dist/root-write-verification.d.ts +11 -0
- package/dist/root-write-verification.d.ts.map +1 -0
- package/dist/root-write-verification.js +91 -0
- package/dist/secret-file.d.ts +1 -6
- package/dist/secret-file.d.ts.map +1 -1
- package/dist/secret-file.js +49 -120
- package/dist/secret-read-async.d.ts +1 -1
- package/dist/secret-read-async.d.ts.map +1 -1
- package/dist/secret-read-async.js +51 -72
- package/dist/secret-read-policy.d.ts +13 -0
- package/dist/secret-read-policy.d.ts.map +1 -0
- package/dist/secret-read-policy.js +28 -0
- package/dist/secret.d.ts +2 -1
- package/dist/secret.d.ts.map +1 -1
- package/dist/secret.js +2 -1
- package/dist/secure-file.d.ts.map +1 -1
- package/dist/secure-file.js +42 -29
- package/dist/sibling-staged-file.d.ts +15 -0
- package/dist/sibling-staged-file.d.ts.map +1 -0
- package/dist/sibling-staged-file.js +148 -0
- package/dist/sibling-temp.d.ts +3 -0
- package/dist/sibling-temp.d.ts.map +1 -1
- package/dist/sibling-temp.js +30 -74
- package/dist/sidecar-lock-acquire.d.ts.map +1 -1
- package/dist/sidecar-lock-acquire.js +48 -27
- package/dist/sidecar-lock-handle.d.ts +6 -2
- package/dist/sidecar-lock-handle.d.ts.map +1 -1
- package/dist/sidecar-lock-handle.js +17 -3
- package/dist/sidecar-lock-policy.d.ts +2 -0
- package/dist/sidecar-lock-policy.d.ts.map +1 -1
- package/dist/sidecar-lock-policy.js +29 -0
- package/dist/sidecar-lock-reclaim.d.ts.map +1 -1
- package/dist/sidecar-lock-reclaim.js +29 -7
- package/dist/sidecar-lock.d.ts.map +1 -1
- package/dist/sidecar-lock.js +28 -16
- package/dist/staged-directory.d.ts +16 -0
- package/dist/staged-directory.d.ts.map +1 -0
- package/dist/staged-directory.js +60 -0
- package/dist/staged-file-types.d.ts +56 -0
- package/dist/staged-file-types.d.ts.map +1 -0
- package/dist/staged-file-types.js +1 -0
- package/dist/staged-file.d.ts +10 -0
- package/dist/staged-file.d.ts.map +1 -0
- package/dist/staged-file.js +15 -0
- package/dist/strict-file-identity.d.ts +6 -0
- package/dist/strict-file-identity.d.ts.map +1 -0
- package/dist/strict-file-identity.js +48 -0
- package/dist/suppressed-error.d.ts +6 -0
- package/dist/suppressed-error.d.ts.map +1 -0
- package/dist/suppressed-error.js +15 -0
- package/dist/temp-cleanup.d.ts +2 -0
- package/dist/temp-cleanup.d.ts.map +1 -1
- package/dist/temp-cleanup.js +25 -10
- package/dist/temp-workspace-owner.d.ts +23 -0
- package/dist/temp-workspace-owner.d.ts.map +1 -0
- package/dist/temp-workspace-owner.js +320 -0
- package/dist/temp.d.ts +1 -1
- package/dist/temp.d.ts.map +1 -1
- package/dist/test-hooks.d.ts +5 -0
- package/dist/test-hooks.d.ts.map +1 -1
- package/dist/windows-owner.d.ts +3 -0
- package/dist/windows-owner.d.ts.map +1 -1
- package/dist/windows-owner.js +10 -2
- package/docs/advanced.md +19 -2
- package/docs/archive.md +250 -35
- package/docs/atomic.md +11 -2
- package/docs/config.md +7 -0
- package/docs/contributing.md +45 -7
- package/docs/durability.md +18 -5
- package/docs/errors.md +16 -1
- package/docs/file-store.md +2 -0
- package/docs/index.md +3 -1
- package/docs/install.md +20 -8
- package/docs/json.md +8 -4
- package/docs/migrating-to-0.5.md +7 -7
- package/docs/migrating-to-0.6.md +43 -0
- package/docs/native-helper.md +30 -8
- package/docs/native.md +73 -17
- package/docs/output.md +10 -0
- package/docs/path-scope.md +28 -2
- package/docs/permissions.md +13 -2
- package/docs/public-api.md +6 -3
- package/docs/quickstart.md +1 -1
- package/docs/reading.md +1 -1
- package/docs/regular-file.md +9 -2
- package/docs/root.md +3 -1
- package/docs/secret-file.md +12 -0
- package/docs/secure-file.md +21 -3
- package/docs/security-model.md +33 -1
- package/docs/sidecar-lock.md +15 -1
- package/docs/staged-file.md +178 -0
- package/docs/store.md +13 -0
- package/docs/temp.md +128 -14
- package/docs/testing.md +1 -1
- package/docs/writing.md +19 -0
- package/package.json +16 -9
- package/dist/native/darwin-arm64/fs-safe-native.node +0 -0
- package/dist/native/darwin-x64/fs-safe-native.node +0 -0
- package/dist/native/linux-arm64-gnu/fs-safe-native.node +0 -0
- package/dist/native/linux-arm64-musl/fs-safe-native.node +0 -0
- package/dist/native/linux-x64-gnu/fs-safe-native.node +0 -0
- package/dist/native/linux-x64-musl/fs-safe-native.node +0 -0
- package/dist/native/win32-x64-msvc/fs-safe-native.node +0 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { syncDirectorySync } from "./directory-durability.js";
|
|
4
5
|
import { FsSafeError } from "./errors.js";
|
|
5
6
|
import { sameFileIdentity } from "./file-identity.js";
|
|
6
7
|
import { assertSyncDirectoryGuard, ensureParentSync, ensureStoreDirectorySync, } from "./file-store-boundary.js";
|
|
@@ -48,15 +49,35 @@ export function writeFileSyncAtomic(params) {
|
|
|
48
49
|
if (parentGuard) {
|
|
49
50
|
assertSyncDirectoryGuard(parentGuard);
|
|
50
51
|
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
52
|
+
const tempStat = (() => {
|
|
53
|
+
const descriptor = fs.openSync(tempPath, "wx", params.mode);
|
|
54
|
+
tempExists = true;
|
|
55
|
+
try {
|
|
56
|
+
fs.writeFileSync(descriptor, params.content);
|
|
57
|
+
try {
|
|
58
|
+
fs.fchmodSync(descriptor, params.mode);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// Best-effort on platforms that do not enforce POSIX modes.
|
|
62
|
+
}
|
|
63
|
+
const opened = fs.fstatSync(descriptor);
|
|
64
|
+
if (!opened.isFile() || opened.nlink > 1) {
|
|
65
|
+
throw new FsSafeError("path-mismatch", "store temp is not an owned regular file");
|
|
66
|
+
}
|
|
67
|
+
fs.fsyncSync(descriptor);
|
|
68
|
+
return opened;
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
fs.closeSync(descriptor);
|
|
72
|
+
}
|
|
73
|
+
})();
|
|
74
|
+
const tempPathStat = fs.lstatSync(tempPath);
|
|
75
|
+
if (tempPathStat.isSymbolicLink() ||
|
|
76
|
+
!tempPathStat.isFile() ||
|
|
77
|
+
tempPathStat.nlink > 1 ||
|
|
78
|
+
!sameFileIdentity(tempPathStat, tempStat)) {
|
|
79
|
+
throw new FsSafeError("path-mismatch", "store temp changed before publication");
|
|
58
80
|
}
|
|
59
|
-
const tempStat = fs.lstatSync(tempPath);
|
|
60
81
|
if (parentGuard) {
|
|
61
82
|
assertSyncDirectoryGuard(parentGuard);
|
|
62
83
|
}
|
|
@@ -82,6 +103,15 @@ export function writeFileSyncAtomic(params) {
|
|
|
82
103
|
cause: error instanceof Error ? error : undefined,
|
|
83
104
|
});
|
|
84
105
|
}
|
|
106
|
+
if (parentGuard) {
|
|
107
|
+
assertSyncDirectoryGuard(parentGuard);
|
|
108
|
+
syncDirectorySync({
|
|
109
|
+
path: parentGuard.dir,
|
|
110
|
+
realPath: parentGuard.realPath,
|
|
111
|
+
identity: parentGuard.stat,
|
|
112
|
+
}, { label: "store parent" });
|
|
113
|
+
assertSyncDirectoryGuard(parentGuard);
|
|
114
|
+
}
|
|
85
115
|
return filePath;
|
|
86
116
|
}
|
|
87
117
|
finally {
|
package/dist/file-store.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-store.d.ts","sourceRoot":"","sources":["../src/file-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"file-store.d.ts","sourceRoot":"","sources":["../src/file-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAK5C,OAAO,EAA4B,KAAK,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC7F,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAOnE,OAAO,EAAmB,KAAK,oBAAoB,EAAE,KAAK,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAItG,OAAO,EAAQ,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,KAAK,IAAI,EAAE,KAAK,eAAe,EAAE,MAAM,WAAW,CAAC;AAOpG,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,eAAe,GAAG;IAAE,QAAQ,CAAC,EAAE,cAAc,CAAA;CAAE,CAAC;AAEnF,MAAM,MAAM,SAAS,GAAG;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,KAAK,CACH,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,GAAG,UAAU,EACzB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAAC;IACnB,WAAW,CACT,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,QAAQ,EAChB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAAC;IACnB,MAAM,CACJ,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAAC;IACnB,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3E,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5E,QAAQ,CACN,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,MAAM,CAAC,CAAC;IACnB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC/F,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACxF,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAC1B,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IACrB,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,SAAS,CACP,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,GAAG,UAAU,EACzB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAAC;IACnB,SAAS,CACP,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,OAAO,EACb,OAAO,CAAC,EAAE,qBAAqB,GAAG;QAAE,eAAe,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9D,OAAO,CAAC,MAAM,CAAC,CAAC;IACnB,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IACtF,YAAY,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,IAAI,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,GAAG,IAAI,CAAC;IACvF,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,CAAC,GAAG,IAAI,CAAC;IAC/F,KAAK,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,MAAM,CAAC;IAChG,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,UAAU,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,MAAM,CAAC;IACpG,SAAS,CACP,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,OAAO,EACb,OAAO,CAAC,EAAE,qBAAqB,GAAG;QAAE,eAAe,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9D,MAAM,CAAC;CACX,CAAC;AAuHF,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,SAAS,CAuM9D;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,gBAAgB,GAAG,aAAa,CAsEtE"}
|
package/dist/file-store.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import syncFs from "node:fs";
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { normalizeMaxBytes } from "./byte-budget.js";
|
|
4
5
|
import { readFileDescriptorBoundedSync } from "./bounded-read.js";
|
|
5
6
|
import { FsSafeError } from "./errors.js";
|
|
7
|
+
import { assertFileStoreMaxBytes } from "./file-store-limit.js";
|
|
6
8
|
import { pruneExpiredStoreEntries } from "./file-store-prune.js";
|
|
7
9
|
import { ensureParentInRoot, openWritableStoreRoot, writeStreamToTempSource, } from "./file-store-boundary.js";
|
|
8
10
|
import { writeFileSyncAtomic } from "./file-store-sync-write.js";
|
|
@@ -38,15 +40,8 @@ function assertRelativePath(relativePath) {
|
|
|
38
40
|
function resolveStorePath(rootDir, relativePath) {
|
|
39
41
|
return resolveSafeRelativePath(rootDir, assertRelativePath(relativePath));
|
|
40
42
|
}
|
|
41
|
-
function assertMaxBytes(size, maxBytes) {
|
|
42
|
-
if (maxBytes !== undefined && size > maxBytes) {
|
|
43
|
-
throw new FsSafeError("too-large", `file exceeds maximum size of ${maxBytes} bytes`);
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
43
|
function isNotFound(error) {
|
|
47
|
-
return error instanceof FsSafeError
|
|
48
|
-
? error.code === "not-found"
|
|
49
|
-
: isNotFoundPathError(error);
|
|
44
|
+
return error instanceof FsSafeError ? error.code === "not-found" : isNotFoundPathError(error);
|
|
50
45
|
}
|
|
51
46
|
function handleSyncStoreReadOpenFailure(opened) {
|
|
52
47
|
return matchRootFileOpenFailure(opened, {
|
|
@@ -81,7 +76,7 @@ async function readFileStoreCopySource(params) {
|
|
|
81
76
|
if (sourceStat.isSymbolicLink() || !sourceStat.isFile()) {
|
|
82
77
|
throw new FsSafeError("not-file", "source path is not a file");
|
|
83
78
|
}
|
|
84
|
-
|
|
79
|
+
assertFileStoreMaxBytes(sourceStat.size, params.maxBytes);
|
|
85
80
|
try {
|
|
86
81
|
return (await readRegularFile({ filePath: params.sourcePath, maxBytes: params.maxBytes }))
|
|
87
82
|
.buffer;
|
|
@@ -108,7 +103,7 @@ async function copyIntoRoot(params) {
|
|
|
108
103
|
if (sourceStat.isSymbolicLink() || !sourceStat.isFile()) {
|
|
109
104
|
throw new FsSafeError("not-file", "source path is not a file");
|
|
110
105
|
}
|
|
111
|
-
|
|
106
|
+
assertFileStoreMaxBytes(sourceStat.size, params.maxBytes);
|
|
112
107
|
const dirMode = params.dirMode ?? 0o700;
|
|
113
108
|
const scopedRoot = await openWritableStoreRoot({
|
|
114
109
|
rootDir: params.rootDir,
|
|
@@ -128,7 +123,7 @@ export function fileStore(options) {
|
|
|
128
123
|
const privateMode = options.private ?? false;
|
|
129
124
|
const dirMode = options.dirMode ?? 0o700;
|
|
130
125
|
const mode = options.mode ?? 0o600;
|
|
131
|
-
const maxBytes = options.maxBytes;
|
|
126
|
+
const maxBytes = normalizeMaxBytes(options.maxBytes);
|
|
132
127
|
async function openRoot() {
|
|
133
128
|
return await root(rootDir, { hardlinks: "reject", maxBytes });
|
|
134
129
|
}
|
|
@@ -136,7 +131,8 @@ export function fileStore(options) {
|
|
|
136
131
|
const safeRelativePath = assertRelativePath(relativePath);
|
|
137
132
|
const destination = resolveStorePath(rootDir, safeRelativePath);
|
|
138
133
|
const content = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
139
|
-
|
|
134
|
+
const writeMaxBytes = normalizeMaxBytes(writeOptions?.maxBytes, { defaultValue: maxBytes });
|
|
135
|
+
assertFileStoreMaxBytes(content.byteLength, writeMaxBytes);
|
|
140
136
|
if (privateMode) {
|
|
141
137
|
await writeSecretFileAtomic({
|
|
142
138
|
rootDir,
|
|
@@ -151,7 +147,7 @@ export function fileStore(options) {
|
|
|
151
147
|
const scopedRoot = await openWritableStoreRoot({
|
|
152
148
|
rootDir,
|
|
153
149
|
dirMode: writeDirMode,
|
|
154
|
-
maxBytes:
|
|
150
|
+
maxBytes: writeMaxBytes,
|
|
155
151
|
});
|
|
156
152
|
await ensureParentInRoot(scopedRoot, safeRelativePath, writeDirMode);
|
|
157
153
|
await scopedRoot.write(safeRelativePath, content, {
|
|
@@ -168,14 +164,15 @@ export function fileStore(options) {
|
|
|
168
164
|
writeStream: async (relativePath, stream, writeOptions) => {
|
|
169
165
|
const safeRelativePath = assertRelativePath(relativePath);
|
|
170
166
|
const destination = resolveStorePath(rootDir, safeRelativePath);
|
|
171
|
-
const
|
|
167
|
+
const configuredLimit = normalizeMaxBytes(writeOptions?.maxBytes, { defaultValue: maxBytes });
|
|
168
|
+
const limit = configuredLimit ?? (privateMode ? DEFAULT_ROOT_MAX_BYTES : undefined);
|
|
172
169
|
if (privateMode) {
|
|
173
170
|
const chunks = [];
|
|
174
171
|
let total = 0;
|
|
175
172
|
for await (const chunk of stream) {
|
|
176
173
|
const buffer = typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk);
|
|
177
174
|
total += buffer.byteLength;
|
|
178
|
-
|
|
175
|
+
assertFileStoreMaxBytes(total, limit);
|
|
179
176
|
chunks.push(buffer);
|
|
180
177
|
}
|
|
181
178
|
await writeSecretFileAtomic({
|
|
@@ -208,23 +205,25 @@ export function fileStore(options) {
|
|
|
208
205
|
}
|
|
209
206
|
return destination;
|
|
210
207
|
},
|
|
211
|
-
copyIn: async (relativePath, sourcePath, writeOptions) =>
|
|
212
|
-
|
|
208
|
+
copyIn: async (relativePath, sourcePath, writeOptions) => {
|
|
209
|
+
const configuredLimit = normalizeMaxBytes(writeOptions?.maxBytes, { defaultValue: maxBytes });
|
|
210
|
+
if (privateMode) {
|
|
213
211
|
const buffer = await readFileStoreCopySource({
|
|
214
212
|
sourcePath,
|
|
215
|
-
maxBytes:
|
|
213
|
+
maxBytes: configuredLimit ?? DEFAULT_ROOT_MAX_BYTES,
|
|
216
214
|
});
|
|
217
215
|
return await write(relativePath, buffer, writeOptions);
|
|
218
|
-
}
|
|
219
|
-
|
|
216
|
+
}
|
|
217
|
+
return await copyIntoRoot({
|
|
220
218
|
rootDir,
|
|
221
219
|
relativePath,
|
|
222
220
|
sourcePath,
|
|
223
221
|
dirMode: writeOptions?.dirMode ?? dirMode,
|
|
224
|
-
maxBytes:
|
|
222
|
+
maxBytes: configuredLimit,
|
|
225
223
|
mode: writeOptions?.mode ?? mode,
|
|
226
224
|
tempPrefix: writeOptions?.tempPrefix,
|
|
227
|
-
})
|
|
225
|
+
});
|
|
226
|
+
},
|
|
228
227
|
open: async (relativePath, readOptions) => await (await openRoot()).open(assertRelativePath(relativePath), readOptions),
|
|
229
228
|
read: async (relativePath, readOptions) => await (await openRoot()).read(assertRelativePath(relativePath), readOptions),
|
|
230
229
|
readBytes: async (relativePath, readOptions) => await (await openRoot()).readBytes(assertRelativePath(relativePath), readOptions),
|
|
@@ -301,11 +300,12 @@ export function fileStoreSync(options) {
|
|
|
301
300
|
const privateMode = options.private ?? false;
|
|
302
301
|
const dirMode = options.dirMode ?? 0o700;
|
|
303
302
|
const mode = options.mode ?? 0o600;
|
|
304
|
-
const maxBytes = options.maxBytes;
|
|
303
|
+
const maxBytes = normalizeMaxBytes(options.maxBytes);
|
|
305
304
|
function write(relativePath, data, writeOptions) {
|
|
306
305
|
const destination = resolveStorePath(rootDir, relativePath);
|
|
307
306
|
const content = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
308
|
-
|
|
307
|
+
const writeMaxBytes = normalizeMaxBytes(writeOptions?.maxBytes, { defaultValue: maxBytes });
|
|
308
|
+
assertFileStoreMaxBytes(content.byteLength, writeMaxBytes);
|
|
309
309
|
return writeFileSyncAtomic({
|
|
310
310
|
rootDir,
|
|
311
311
|
filePath: destination,
|
|
@@ -319,6 +319,7 @@ export function fileStoreSync(options) {
|
|
|
319
319
|
rootDir,
|
|
320
320
|
path: (relativePath) => resolveStorePath(rootDir, relativePath),
|
|
321
321
|
readTextIfExists: (relativePath, readOptions) => {
|
|
322
|
+
const limit = normalizeMaxBytes(readOptions?.maxBytes, { defaultValue: maxBytes });
|
|
322
323
|
const targetPath = resolveStorePath(rootDir, relativePath);
|
|
323
324
|
const opened = openRootFileSync({
|
|
324
325
|
absolutePath: targetPath,
|
|
@@ -330,8 +331,7 @@ export function fileStoreSync(options) {
|
|
|
330
331
|
return handleSyncStoreReadOpenFailure(opened);
|
|
331
332
|
}
|
|
332
333
|
try {
|
|
333
|
-
|
|
334
|
-
const limit = readOptions?.maxBytes ?? maxBytes;
|
|
334
|
+
assertFileStoreMaxBytes(opened.stat.size, limit);
|
|
335
335
|
try {
|
|
336
336
|
return limit === undefined
|
|
337
337
|
? syncFs.readFileSync(opened.fd, "utf8")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-durable-queue-directory.d.ts","sourceRoot":"","sources":["../src/json-durable-queue-directory.ts"],"names":[],"mappings":"AAKA,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAiBf"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { syncDirectory } from "./directory-durability.js";
|
|
4
|
+
import { isPathRelativeEscape } from "./path.js";
|
|
5
|
+
export async function syncQueueDirectoryCreation(dir, validationBase) {
|
|
6
|
+
const baseReal = await fs.realpath(validationBase);
|
|
7
|
+
const targetReal = await fs.realpath(dir);
|
|
8
|
+
const relative = path.relative(baseReal, targetReal);
|
|
9
|
+
if (isPathRelativeEscape(relative)) {
|
|
10
|
+
throw new Error(`durable queue directory escapes validation base: ${dir}`);
|
|
11
|
+
}
|
|
12
|
+
const directories = [baseReal];
|
|
13
|
+
for (const segment of relative.split(path.sep).filter(Boolean)) {
|
|
14
|
+
directories.push(path.join(directories.at(-1), segment));
|
|
15
|
+
}
|
|
16
|
+
await syncDirectory(targetReal, { label: "durable queue directory" });
|
|
17
|
+
for (const parent of directories.slice(0, -1).toReversed()) {
|
|
18
|
+
await syncDirectory(parent, { label: "durable queue parent" });
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type DurableQueueEntryPathsLike = {
|
|
2
|
+
jsonPath: string;
|
|
3
|
+
deliveredPath: string;
|
|
4
|
+
processingPath?: string;
|
|
5
|
+
};
|
|
6
|
+
export declare function durableQueueProcessingPath(paths: DurableQueueEntryPathsLike): string;
|
|
7
|
+
export declare function claimDurableQueueEntry(paths: DurableQueueEntryPathsLike): Promise<string | null>;
|
|
8
|
+
export declare function completeDeliveredQueueEntry(paths: DurableQueueEntryPathsLike): Promise<boolean>;
|
|
9
|
+
export declare function acknowledgeDurableQueueEntry(paths: DurableQueueEntryPathsLike): Promise<void>;
|
|
10
|
+
export declare function moveDurableQueueEntryToFailed(params: {
|
|
11
|
+
paths: DurableQueueEntryPathsLike;
|
|
12
|
+
failedPath: string;
|
|
13
|
+
}): Promise<void>;
|
|
14
|
+
//# sourceMappingURL=json-durable-queue-ownership.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-durable-queue-ownership.d.ts","sourceRoot":"","sources":["../src/json-durable-queue-ownership.ts"],"names":[],"mappings":"AAaA,MAAM,MAAM,0BAA0B,GAAG;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAQF,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,0BAA0B,GAAG,MAAM,CAKpF;AAqED,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,0BAA0B,GAChC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAGxB;AAED,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,0BAA0B,GAChC,OAAO,CAAC,OAAO,CAAC,CAalB;AAED,wBAAsB,4BAA4B,CAChD,KAAK,EAAE,0BAA0B,GAChC,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,6BAA6B,CAAC,MAAM,EAAE;IAC1D,KAAK,EAAE,0BAA0B,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiDhB"}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { syncDirectory } from "./directory-durability.js";
|
|
4
|
+
import { FsSafeError } from "./errors.js";
|
|
5
|
+
import { sameFileIdentityForCleanup } from "./file-identity.js";
|
|
6
|
+
import { recoverDurableQueueRetirement, retireDurableQueueSource, } from "./json-durable-queue-retirement.js";
|
|
7
|
+
import { withQueueTransferLock } from "./json-durable-queue-transfer-lock.js";
|
|
8
|
+
import { serializePathWrite } from "./write-queue.js";
|
|
9
|
+
function getErrorCode(error) {
|
|
10
|
+
return error && typeof error === "object" && "code" in error
|
|
11
|
+
? String(error.code)
|
|
12
|
+
: null;
|
|
13
|
+
}
|
|
14
|
+
export function durableQueueProcessingPath(paths) {
|
|
15
|
+
if (paths.processingPath)
|
|
16
|
+
return paths.processingPath;
|
|
17
|
+
return paths.jsonPath.endsWith(".json")
|
|
18
|
+
? `${paths.jsonPath.slice(0, -".json".length)}.processing`
|
|
19
|
+
: `${paths.jsonPath}.processing`;
|
|
20
|
+
}
|
|
21
|
+
async function lstatOrNull(filePath) {
|
|
22
|
+
try {
|
|
23
|
+
return await fs.lstat(filePath, { bigint: true });
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
if (getErrorCode(error) === "ENOENT")
|
|
27
|
+
return null;
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async function regularQueueFileIdentity(filePath) {
|
|
32
|
+
const identity = await lstatOrNull(filePath);
|
|
33
|
+
if (!identity)
|
|
34
|
+
return null;
|
|
35
|
+
if (identity.isSymbolicLink() || !identity.isFile()) {
|
|
36
|
+
throw new Error(`queue entry is not a regular file: ${filePath}`);
|
|
37
|
+
}
|
|
38
|
+
return identity;
|
|
39
|
+
}
|
|
40
|
+
async function withQueueEntryLock(paths, run) {
|
|
41
|
+
return await serializePathWrite(paths.jsonPath, async () => await withQueueTransferLock(path.resolve(paths.jsonPath), run));
|
|
42
|
+
}
|
|
43
|
+
async function claimDurableQueueEntryUnlocked(paths) {
|
|
44
|
+
const processingPath = durableQueueProcessingPath(paths);
|
|
45
|
+
await recoverDurableQueueRetirement({ jsonPath: paths.jsonPath, processingPath });
|
|
46
|
+
const existingProcessing = await regularQueueFileIdentity(processingPath);
|
|
47
|
+
if (existingProcessing) {
|
|
48
|
+
const pending = await lstatOrNull(paths.jsonPath);
|
|
49
|
+
if (pending &&
|
|
50
|
+
!pending.isSymbolicLink() &&
|
|
51
|
+
pending.isFile() &&
|
|
52
|
+
sameFileIdentityForCleanup(pending, existingProcessing)) {
|
|
53
|
+
await retireDurableQueueSource({ jsonPath: paths.jsonPath, processingPath });
|
|
54
|
+
}
|
|
55
|
+
return processingPath;
|
|
56
|
+
}
|
|
57
|
+
const pending = await lstatOrNull(paths.jsonPath);
|
|
58
|
+
if (!pending || pending.isSymbolicLink() || !pending.isFile())
|
|
59
|
+
return null;
|
|
60
|
+
if (pending.nlink > 1n || !sameFileIdentityForCleanup(pending, pending)) {
|
|
61
|
+
throw new FsSafeError("path-mismatch", "queue entry is not exclusively owned");
|
|
62
|
+
}
|
|
63
|
+
try {
|
|
64
|
+
await fs.link(paths.jsonPath, processingPath);
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
if (getErrorCode(error) === "EEXIST") {
|
|
68
|
+
return (await regularQueueFileIdentity(processingPath)) ? processingPath : null;
|
|
69
|
+
}
|
|
70
|
+
if (getErrorCode(error) === "ENOENT")
|
|
71
|
+
return null;
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
await syncDirectory(path.dirname(processingPath));
|
|
75
|
+
const claimed = await regularQueueFileIdentity(processingPath);
|
|
76
|
+
if (!claimed || claimed.nlink > 2n) {
|
|
77
|
+
throw new FsSafeError("path-mismatch", "queue claim is not an owned regular file");
|
|
78
|
+
}
|
|
79
|
+
await retireDurableQueueSource({ jsonPath: paths.jsonPath, processingPath });
|
|
80
|
+
return processingPath;
|
|
81
|
+
}
|
|
82
|
+
export async function claimDurableQueueEntry(paths) {
|
|
83
|
+
if (!(await lstatOrNull(path.dirname(paths.jsonPath))))
|
|
84
|
+
return null;
|
|
85
|
+
return await withQueueEntryLock(paths, async () => await claimDurableQueueEntryUnlocked(paths));
|
|
86
|
+
}
|
|
87
|
+
export async function completeDeliveredQueueEntry(paths) {
|
|
88
|
+
if (!(await lstatOrNull(path.dirname(paths.deliveredPath))))
|
|
89
|
+
return false;
|
|
90
|
+
return await withQueueEntryLock(paths, async () => {
|
|
91
|
+
if (!(await regularQueueFileIdentity(paths.deliveredPath)))
|
|
92
|
+
return false;
|
|
93
|
+
if (await regularQueueFileIdentity(durableQueueProcessingPath(paths))) {
|
|
94
|
+
await fs.unlink(paths.deliveredPath);
|
|
95
|
+
await syncDirectory(path.dirname(paths.deliveredPath));
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
await fs.unlink(paths.deliveredPath);
|
|
99
|
+
await syncDirectory(path.dirname(paths.deliveredPath));
|
|
100
|
+
return true;
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
export async function acknowledgeDurableQueueEntry(paths) {
|
|
104
|
+
if (!(await lstatOrNull(path.dirname(paths.jsonPath))))
|
|
105
|
+
return;
|
|
106
|
+
await withQueueEntryLock(paths, async () => {
|
|
107
|
+
const processingPath = durableQueueProcessingPath(paths);
|
|
108
|
+
const processing = await regularQueueFileIdentity(processingPath);
|
|
109
|
+
const delivered = await regularQueueFileIdentity(paths.deliveredPath);
|
|
110
|
+
if (!processing) {
|
|
111
|
+
if (delivered) {
|
|
112
|
+
await fs.unlink(paths.deliveredPath);
|
|
113
|
+
await syncDirectory(path.dirname(paths.deliveredPath));
|
|
114
|
+
}
|
|
115
|
+
if (await regularQueueFileIdentity(paths.jsonPath)) {
|
|
116
|
+
throw new FsSafeError("path-mismatch", "queue acknowledgement requires a processing claim");
|
|
117
|
+
}
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
if (delivered)
|
|
121
|
+
await fs.unlink(paths.deliveredPath);
|
|
122
|
+
await fs.rename(processingPath, paths.deliveredPath);
|
|
123
|
+
await syncDirectory(path.dirname(paths.deliveredPath));
|
|
124
|
+
await fs.unlink(paths.deliveredPath);
|
|
125
|
+
await syncDirectory(path.dirname(paths.deliveredPath));
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
export async function moveDurableQueueEntryToFailed(params) {
|
|
129
|
+
await withQueueEntryLock(params.paths, async () => {
|
|
130
|
+
const processingPath = durableQueueProcessingPath(params.paths);
|
|
131
|
+
const processing = await regularQueueFileIdentity(processingPath);
|
|
132
|
+
const existingFailed = await regularQueueFileIdentity(params.failedPath);
|
|
133
|
+
if (!processing && existingFailed) {
|
|
134
|
+
await syncDirectory(path.dirname(params.failedPath));
|
|
135
|
+
await syncDirectory(path.dirname(processingPath));
|
|
136
|
+
const recoveredFailed = await regularQueueFileIdentity(params.failedPath);
|
|
137
|
+
if (!recoveredFailed ||
|
|
138
|
+
!sameFileIdentityForCleanup(existingFailed, recoveredFailed)) {
|
|
139
|
+
throw new FsSafeError("path-mismatch", "failed queue destination changed during recovery");
|
|
140
|
+
}
|
|
141
|
+
if (await regularQueueFileIdentity(params.paths.jsonPath)) {
|
|
142
|
+
throw new FsSafeError("already-exists", "failed queue destination already exists");
|
|
143
|
+
}
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
const sourcePath = processing
|
|
147
|
+
? processingPath
|
|
148
|
+
: await claimDurableQueueEntryUnlocked(params.paths);
|
|
149
|
+
if (!sourcePath) {
|
|
150
|
+
throw Object.assign(new Error("queue entry does not exist"), { code: "ENOENT" });
|
|
151
|
+
}
|
|
152
|
+
const source = await regularQueueFileIdentity(sourcePath);
|
|
153
|
+
const failed = await regularQueueFileIdentity(params.failedPath);
|
|
154
|
+
if (failed) {
|
|
155
|
+
if (source && sameFileIdentityForCleanup(source, failed)) {
|
|
156
|
+
await syncDirectory(path.dirname(params.failedPath));
|
|
157
|
+
await fs.unlink(sourcePath);
|
|
158
|
+
await syncDirectory(path.dirname(sourcePath));
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
throw new FsSafeError("already-exists", "failed queue destination already exists");
|
|
162
|
+
}
|
|
163
|
+
await fs.link(sourcePath, params.failedPath);
|
|
164
|
+
await syncDirectory(path.dirname(params.failedPath));
|
|
165
|
+
await fs.unlink(sourcePath);
|
|
166
|
+
await syncDirectory(path.dirname(sourcePath));
|
|
167
|
+
});
|
|
168
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare function recoverDurableQueueRetirement(params: {
|
|
2
|
+
jsonPath: string;
|
|
3
|
+
processingPath: string;
|
|
4
|
+
}): Promise<void>;
|
|
5
|
+
export declare function retireDurableQueueSource(params: {
|
|
6
|
+
jsonPath: string;
|
|
7
|
+
processingPath: string;
|
|
8
|
+
}): Promise<void>;
|
|
9
|
+
//# sourceMappingURL=json-durable-queue-retirement.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-durable-queue-retirement.d.ts","sourceRoot":"","sources":["../src/json-durable-queue-retirement.ts"],"names":[],"mappings":"AAgFA,wBAAsB,6BAA6B,CAAC,MAAM,EAAE;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmChB;AAED,wBAAsB,wBAAwB,CAAC,MAAM,EAAE;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBhB"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { syncDirectory } from "./directory-durability.js";
|
|
4
|
+
import { FsSafeError } from "./errors.js";
|
|
5
|
+
import { sameFileIdentityForCleanup } from "./file-identity.js";
|
|
6
|
+
const RETIREMENT_ROOT_NAME = ".fs-safe-retirements";
|
|
7
|
+
const RETIREMENT_ENTRY_NAME = "entry";
|
|
8
|
+
function getErrorCode(error) {
|
|
9
|
+
return error && typeof error === "object" && "code" in error
|
|
10
|
+
? String(error.code)
|
|
11
|
+
: null;
|
|
12
|
+
}
|
|
13
|
+
async function lstatOrNull(filePath) {
|
|
14
|
+
try {
|
|
15
|
+
return await fs.lstat(filePath, { bigint: true });
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
if (getErrorCode(error) === "ENOENT")
|
|
19
|
+
return null;
|
|
20
|
+
throw error;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function assertDirectory(identity, label) {
|
|
24
|
+
if (identity.isSymbolicLink() || !identity.isDirectory()) {
|
|
25
|
+
throw new FsSafeError("path-mismatch", `${label} is not an owned directory`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
async function regularFileIdentity(filePath) {
|
|
29
|
+
const identity = await lstatOrNull(filePath);
|
|
30
|
+
if (!identity)
|
|
31
|
+
return null;
|
|
32
|
+
if (identity.isSymbolicLink() || !identity.isFile()) {
|
|
33
|
+
throw new FsSafeError("path-mismatch", "queue retirement path is not a regular file");
|
|
34
|
+
}
|
|
35
|
+
return identity;
|
|
36
|
+
}
|
|
37
|
+
function retirementPaths(jsonPath) {
|
|
38
|
+
const rootPath = path.join(path.dirname(jsonPath), RETIREMENT_ROOT_NAME);
|
|
39
|
+
const dirPath = path.join(rootPath, path.basename(jsonPath));
|
|
40
|
+
return { dirPath, entryPath: path.join(dirPath, RETIREMENT_ENTRY_NAME), rootPath };
|
|
41
|
+
}
|
|
42
|
+
async function inspectRetirementRoot(jsonPath) {
|
|
43
|
+
const rootPath = retirementPaths(jsonPath).rootPath;
|
|
44
|
+
const root = await lstatOrNull(rootPath);
|
|
45
|
+
if (!root)
|
|
46
|
+
return null;
|
|
47
|
+
assertDirectory(root, "queue retirement root");
|
|
48
|
+
return rootPath;
|
|
49
|
+
}
|
|
50
|
+
async function ensureRetirementRoot(jsonPath) {
|
|
51
|
+
const parentPath = path.dirname(jsonPath);
|
|
52
|
+
const rootPath = retirementPaths(jsonPath).rootPath;
|
|
53
|
+
await fs.mkdir(rootPath, { mode: 0o700 }).catch((error) => {
|
|
54
|
+
if (getErrorCode(error) !== "EEXIST")
|
|
55
|
+
throw error;
|
|
56
|
+
});
|
|
57
|
+
const root = await fs.lstat(rootPath, { bigint: true });
|
|
58
|
+
assertDirectory(root, "queue retirement root");
|
|
59
|
+
await syncDirectory(parentPath);
|
|
60
|
+
return rootPath;
|
|
61
|
+
}
|
|
62
|
+
async function removeRetirementRecord(params) {
|
|
63
|
+
await fs.rmdir(params.dirPath).catch((error) => {
|
|
64
|
+
if (getErrorCode(error) !== "ENOENT")
|
|
65
|
+
throw error;
|
|
66
|
+
});
|
|
67
|
+
await syncDirectory(params.rootPath);
|
|
68
|
+
}
|
|
69
|
+
export async function recoverDurableQueueRetirement(params) {
|
|
70
|
+
const rootPath = await inspectRetirementRoot(params.jsonPath);
|
|
71
|
+
if (!rootPath)
|
|
72
|
+
return;
|
|
73
|
+
const { dirPath, entryPath } = retirementPaths(params.jsonPath);
|
|
74
|
+
const dir = await lstatOrNull(dirPath);
|
|
75
|
+
if (!dir)
|
|
76
|
+
return;
|
|
77
|
+
assertDirectory(dir, "queue retirement record");
|
|
78
|
+
const names = await fs.readdir(dirPath);
|
|
79
|
+
if (names.length === 0) {
|
|
80
|
+
await removeRetirementRecord({ dirPath, rootPath });
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
if (names.length !== 1 || names[0] !== RETIREMENT_ENTRY_NAME) {
|
|
84
|
+
throw new FsSafeError("path-mismatch", "queue retirement record is ambiguous");
|
|
85
|
+
}
|
|
86
|
+
const entry = await regularFileIdentity(entryPath);
|
|
87
|
+
const processing = await regularFileIdentity(params.processingPath);
|
|
88
|
+
if (!entry || !processing) {
|
|
89
|
+
throw new FsSafeError("path-mismatch", "queue retirement ownership is incomplete");
|
|
90
|
+
}
|
|
91
|
+
if (!sameFileIdentityForCleanup(entry, processing)) {
|
|
92
|
+
const pending = await regularFileIdentity(params.jsonPath);
|
|
93
|
+
if (!pending) {
|
|
94
|
+
await fs.link(entryPath, params.jsonPath);
|
|
95
|
+
await syncDirectory(path.dirname(params.jsonPath));
|
|
96
|
+
}
|
|
97
|
+
else if (sameFileIdentityForCleanup(pending, entry)) {
|
|
98
|
+
await syncDirectory(path.dirname(params.jsonPath));
|
|
99
|
+
}
|
|
100
|
+
else if (pending.nlink > 1n) {
|
|
101
|
+
throw new FsSafeError("path-mismatch", "queue replacement identity is ambiguous");
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
await fs.unlink(entryPath);
|
|
105
|
+
await syncDirectory(dirPath);
|
|
106
|
+
await removeRetirementRecord({ dirPath, rootPath });
|
|
107
|
+
}
|
|
108
|
+
export async function retireDurableQueueSource(params) {
|
|
109
|
+
await recoverDurableQueueRetirement(params);
|
|
110
|
+
const rootPath = await ensureRetirementRoot(params.jsonPath);
|
|
111
|
+
const { dirPath, entryPath } = retirementPaths(params.jsonPath);
|
|
112
|
+
await fs.mkdir(dirPath, { mode: 0o700 });
|
|
113
|
+
await syncDirectory(rootPath);
|
|
114
|
+
try {
|
|
115
|
+
await fs.rename(params.jsonPath, entryPath);
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
if (getErrorCode(error) !== "ENOENT")
|
|
119
|
+
throw error;
|
|
120
|
+
await removeRetirementRecord({ dirPath, rootPath });
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
await syncDirectory(dirPath);
|
|
124
|
+
await syncDirectory(path.dirname(params.jsonPath));
|
|
125
|
+
await recoverDurableQueueRetirement(params);
|
|
126
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-durable-queue-transfer-lock.d.ts","sourceRoot":"","sources":["../src/json-durable-queue-transfer-lock.ts"],"names":[],"mappings":"AAKA,wBAAsB,qBAAqB,CAAC,CAAC,EAC3C,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACpB,OAAO,CAAC,CAAC,CAAC,CAqBZ"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { sha256Hex } from "./file-identity.js";
|
|
4
|
+
import { withSidecarLock } from "./sidecar-lock.js";
|
|
5
|
+
export async function withQueueTransferLock(filePath, run) {
|
|
6
|
+
const lockPath = path.join(path.dirname(filePath), `.fs-safe-transfer-${sha256Hex(path.basename(filePath)).slice(0, 32)}.lock`);
|
|
7
|
+
return await withSidecarLock(filePath, {
|
|
8
|
+
managerKey: "fs-safe.queue-transfer",
|
|
9
|
+
lockPath,
|
|
10
|
+
staleMs: 30_000,
|
|
11
|
+
staleRecovery: "fail-closed",
|
|
12
|
+
timeoutMs: 45_000,
|
|
13
|
+
payload: () => ({
|
|
14
|
+
ownerToken: randomUUID(),
|
|
15
|
+
createdAt: new Date().toISOString(),
|
|
16
|
+
}),
|
|
17
|
+
retry: { retries: 180, minTimeout: 25, maxTimeout: 250, factor: 1.1 },
|
|
18
|
+
}, run);
|
|
19
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"json-durable-queue.d.ts","sourceRoot":"","sources":["../src/json-durable-queue.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"json-durable-queue.d.ts","sourceRoot":"","sources":["../src/json-durable-queue.ts"],"names":[],"mappings":"AAiBA,MAAM,MAAM,0BAA0B,GAAG;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,0BAA0B,CAAC,CAAC,IAAI;IAC1C,KAAK,EAAE,CAAC,CAAC;IACT,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,2BAA2B,CAAC,CAAC,IAAI;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAOF,eAAO,MAAM,0CAA0C,QAAmB,CAAC;AAY3E,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtE;AAED,wBAAsB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAUpF;AAmBD,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,MAAM,EAChB,EAAE,EAAE,MAAM,GACT,0BAA0B,CAO5B;AAED,wBAAsB,0BAA0B,CAAC,MAAM,EAAE;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,IAAI,CAAC,CAIhB;AAgLD,wBAAsB,0BAA0B,CAAC,MAAM,EAAE;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,IAAI,CAAC,CAShB;AA4ED,wBAAsB,yBAAyB,CAAC,CAAC,EAC/C,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,OAAO,CAAC,CAAC,CAAC,CASZ;AAED,wBAAsB,wBAAwB,CAAC,KAAK,EAAE,0BAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/F;AAED,wBAAsB,yBAAyB,CAAC,CAAC,EAAE,MAAM,EAAE;IACzD,KAAK,EAAE,0BAA0B,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAsBpB;AAED,wBAAsB,kCAAkC,CAAC,CAAC,EACxD,OAAO,EAAE,2BAA2B,CAAC,CAAC,CAAC,GACtC,OAAO,CAAC,CAAC,EAAE,CAAC,CA4Dd;AAED,wBAAsB,iCAAiC,CAAC,MAAM,EAAE;IAC9D,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;CACZ,GAAG,OAAO,CAAC,IAAI,CAAC,CAShB"}
|