@procedures/fs 0.27.2 → 0.27.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +63 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/files/atomic.d.ts +39 -0
- package/dist/files/atomic.d.ts.map +1 -0
- package/dist/files/atomic.js +164 -0
- package/dist/files/atomic.js.map +1 -0
- package/dist/files/edit.d.ts +36 -0
- package/dist/files/edit.d.ts.map +1 -0
- package/dist/files/edit.js +56 -0
- package/dist/files/edit.js.map +1 -0
- package/dist/files/index.d.ts +14 -0
- package/dist/files/index.d.ts.map +1 -0
- package/dist/files/index.js +14 -0
- package/dist/files/index.js.map +1 -0
- package/dist/files/lock.d.ts +72 -0
- package/dist/files/lock.d.ts.map +1 -0
- package/dist/files/lock.js +106 -0
- package/dist/files/lock.js.map +1 -0
- package/dist/files/ndjson-tail.d.ts +41 -0
- package/dist/files/ndjson-tail.d.ts.map +1 -0
- package/dist/files/ndjson-tail.js +104 -0
- package/dist/files/ndjson-tail.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/package.json +6 -2
- package/src/files/atomic.ts +179 -0
- package/src/files/edit.ts +98 -0
- package/src/files/index.ts +21 -0
- package/src/files/lock.ts +159 -0
- package/src/files/ndjson-tail.ts +124 -0
- package/src/index.ts +2 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The atomic write of a file (roadmap O51).
|
|
3
|
+
*
|
|
4
|
+
* `writeFileAtomic()` writes the data to a new temporary file in the folder of the target. Then it
|
|
5
|
+
* renames the temporary file over the target. Thus a reader sees the old file or the whole new
|
|
6
|
+
* file, also after a crash.
|
|
7
|
+
*/
|
|
8
|
+
/** The options of `writeFileAtomic()`. */
|
|
9
|
+
export interface WriteFileAtomicOptions {
|
|
10
|
+
/** The encoding of a string `data`. The default is `utf8`. */
|
|
11
|
+
encoding?: BufferEncoding | undefined;
|
|
12
|
+
/**
|
|
13
|
+
* The permissions of the file. Without it, the file keeps the permissions of the target. A new
|
|
14
|
+
* file gets `0o666` less the umask.
|
|
15
|
+
*/
|
|
16
|
+
mode?: number | undefined;
|
|
17
|
+
/** With `false`, the function does not flush the file and its folder to the disk. The default is `true`. */
|
|
18
|
+
fsync?: boolean | undefined;
|
|
19
|
+
/**
|
|
20
|
+
* The time in milliseconds to try a rename again after `EPERM`, `EACCES` or `EBUSY`. The default
|
|
21
|
+
* is 2000 on Windows and 0 on the other platforms.
|
|
22
|
+
*/
|
|
23
|
+
renameRetryMs?: number | undefined;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* This function writes `data` to `path` in one step for each reader. A reader sees the old file or
|
|
27
|
+
* the whole new file. It does not see a part of a file at any time.
|
|
28
|
+
*
|
|
29
|
+
* - The function writes a new temporary file in the folder of the target and flushes it to the disk.
|
|
30
|
+
* - Then it renames the temporary file over the target and flushes the folder. Windows gets no flush of the folder.
|
|
31
|
+
* - On Windows, an open handle of the target blocks the rename. Then the function tries the rename again for `renameRetryMs`.
|
|
32
|
+
* - After a failure, the function removes the temporary file and throws the error.
|
|
33
|
+
* - The file keeps the permissions of the target, unless `options.mode` is set. It does not keep the owner.
|
|
34
|
+
* - When `path` is a symlink, the function replaces the target of the link.
|
|
35
|
+
*
|
|
36
|
+
* Two writes at the same time do not mix: the file gets the data of one of them. The folder must exist.
|
|
37
|
+
*/
|
|
38
|
+
export declare function writeFileAtomic(path: string, data: string | Uint8Array, options?: WriteFileAtomicOptions): Promise<void>;
|
|
39
|
+
//# sourceMappingURL=atomic.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"atomic.d.ts","sourceRoot":"","sources":["../../src/files/atomic.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAOH,0CAA0C;AAC1C,MAAM,WAAW,sBAAsB;IACrC,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,cAAc,GAAG,SAAS,CAAC;IACtC;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,4GAA4G;IAC5G,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAuGD;;;;;;;;;;;;GAYG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GAAG,UAAU,EACzB,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,IAAI,CAAC,CA6Bf"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The atomic write of a file (roadmap O51).
|
|
3
|
+
*
|
|
4
|
+
* `writeFileAtomic()` writes the data to a new temporary file in the folder of the target. Then it
|
|
5
|
+
* renames the temporary file over the target. Thus a reader sees the old file or the whole new
|
|
6
|
+
* file, also after a crash.
|
|
7
|
+
*/
|
|
8
|
+
import { randomBytes } from "node:crypto";
|
|
9
|
+
import { chmod, open, realpath, rename, stat, unlink } from "node:fs/promises";
|
|
10
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
11
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
12
|
+
/** On Windows, an antivirus or an indexer that holds the target open gives these errors for a short time. */
|
|
13
|
+
const LOCKED = new Set(["EPERM", "EACCES", "EBUSY"]);
|
|
14
|
+
/** A file system that cannot flush a folder gives these errors. The write is complete without the flush. */
|
|
15
|
+
const NO_FOLDER_SYNC = new Set(["EINVAL", "ENOTSUP", "EISDIR", "EBADF", "EPERM", "EACCES"]);
|
|
16
|
+
/** True on Windows. The function reads the platform at each call, so a test can change it. */
|
|
17
|
+
function isWindows() {
|
|
18
|
+
return process.platform === "win32";
|
|
19
|
+
}
|
|
20
|
+
/** The code of a Node.js error, or an empty string. */
|
|
21
|
+
function codeOf(error) {
|
|
22
|
+
return error?.code ?? "";
|
|
23
|
+
}
|
|
24
|
+
/** True for the error of a path that does not exist. */
|
|
25
|
+
function isMissing(error) {
|
|
26
|
+
const code = codeOf(error);
|
|
27
|
+
return code === "ENOENT" || code === "ENOTDIR";
|
|
28
|
+
}
|
|
29
|
+
/** The absolute path of the file to replace. The function follows a symlink, so the link stays a link. */
|
|
30
|
+
async function targetOf(path) {
|
|
31
|
+
const absolute = resolve(path);
|
|
32
|
+
try {
|
|
33
|
+
return await realpath(absolute);
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
if (isMissing(error))
|
|
37
|
+
return absolute;
|
|
38
|
+
throw error;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/** The permission bits of the target, or undefined when the target does not exist. */
|
|
42
|
+
async function modeOf(target) {
|
|
43
|
+
try {
|
|
44
|
+
return (await stat(target)).mode & 0o7777;
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (isMissing(error))
|
|
48
|
+
return undefined;
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* A new path next to the target. The name starts with a dot and ends in `.tmp`, so a reader that
|
|
54
|
+
* looks for the extension of the target does not see it. The name keeps 32 characters of the
|
|
55
|
+
* target, so it does not become too long.
|
|
56
|
+
*/
|
|
57
|
+
function tempPathOf(target) {
|
|
58
|
+
const name = [...basename(target)].slice(0, 32).join("");
|
|
59
|
+
return join(dirname(target), `.${name}.${randomBytes(8).toString("hex")}.tmp`);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The rename, tried again with a longer wait each time until `retryMs` is over. A random part of
|
|
63
|
+
* each wait keeps two writers of one target out of step.
|
|
64
|
+
*/
|
|
65
|
+
async function renameWithRetry(from, to, retryMs) {
|
|
66
|
+
const deadline = Date.now() + retryMs;
|
|
67
|
+
for (let step = 10;; step = Math.min(step * 2, 100)) {
|
|
68
|
+
try {
|
|
69
|
+
await rename(from, to);
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
const delay = Math.round(step * (0.5 + Math.random()));
|
|
74
|
+
if (!LOCKED.has(codeOf(error)) || Date.now() + delay > deadline)
|
|
75
|
+
throw error;
|
|
76
|
+
await sleep(delay);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/** The removal of the temporary file after a failure. The original error is more important than an error here. */
|
|
81
|
+
async function removeTemp(temp) {
|
|
82
|
+
try {
|
|
83
|
+
await unlink(temp);
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
if (codeOf(error) !== "EPERM")
|
|
87
|
+
return;
|
|
88
|
+
// Windows does not remove a read-only file, so the function clears the flag and tries again
|
|
89
|
+
try {
|
|
90
|
+
await chmod(temp, 0o666);
|
|
91
|
+
await unlink(temp);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// The temporary file stays. The error of the write is the error that the caller gets.
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** The flush of the folder, so that the rename is on the disk. Windows cannot open a folder for it. */
|
|
99
|
+
async function syncFolder(folder) {
|
|
100
|
+
if (isWindows())
|
|
101
|
+
return;
|
|
102
|
+
let handle;
|
|
103
|
+
try {
|
|
104
|
+
handle = await open(folder, "r");
|
|
105
|
+
await handle.sync();
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
if (!NO_FOLDER_SYNC.has(codeOf(error)))
|
|
109
|
+
throw error;
|
|
110
|
+
}
|
|
111
|
+
finally {
|
|
112
|
+
await handle?.close();
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* This function writes `data` to `path` in one step for each reader. A reader sees the old file or
|
|
117
|
+
* the whole new file. It does not see a part of a file at any time.
|
|
118
|
+
*
|
|
119
|
+
* - The function writes a new temporary file in the folder of the target and flushes it to the disk.
|
|
120
|
+
* - Then it renames the temporary file over the target and flushes the folder. Windows gets no flush of the folder.
|
|
121
|
+
* - On Windows, an open handle of the target blocks the rename. Then the function tries the rename again for `renameRetryMs`.
|
|
122
|
+
* - After a failure, the function removes the temporary file and throws the error.
|
|
123
|
+
* - The file keeps the permissions of the target, unless `options.mode` is set. It does not keep the owner.
|
|
124
|
+
* - When `path` is a symlink, the function replaces the target of the link.
|
|
125
|
+
*
|
|
126
|
+
* Two writes at the same time do not mix: the file gets the data of one of them. The folder must exist.
|
|
127
|
+
*/
|
|
128
|
+
export async function writeFileAtomic(path, data, options = {}) {
|
|
129
|
+
const target = await targetOf(path);
|
|
130
|
+
const mode = options.mode ?? (await modeOf(target));
|
|
131
|
+
const flush = options.fsync ?? true;
|
|
132
|
+
const temp = tempPathOf(target);
|
|
133
|
+
let created = false;
|
|
134
|
+
let handle;
|
|
135
|
+
try {
|
|
136
|
+
handle = await open(temp, "wx", mode ?? 0o666);
|
|
137
|
+
created = true;
|
|
138
|
+
await handle.writeFile(data, { encoding: options.encoding ?? "utf8" });
|
|
139
|
+
// The umask changes the mode of open(), so the function sets the mode again
|
|
140
|
+
if (mode !== undefined)
|
|
141
|
+
await handle.chmod(mode);
|
|
142
|
+
if (flush)
|
|
143
|
+
await handle.sync();
|
|
144
|
+
await handle.close();
|
|
145
|
+
handle = undefined;
|
|
146
|
+
await renameWithRetry(temp, target, options.renameRetryMs ?? (isWindows() ? 2000 : 0));
|
|
147
|
+
}
|
|
148
|
+
catch (error) {
|
|
149
|
+
if (handle) {
|
|
150
|
+
try {
|
|
151
|
+
await handle.close();
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
// The error of the write is the error that the caller gets
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
if (created)
|
|
158
|
+
await removeTemp(temp);
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
161
|
+
if (flush)
|
|
162
|
+
await syncFolder(dirname(target));
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=atomic.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"atomic.js","sourceRoot":"","sources":["../../src/files/atomic.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAmB,MAAM,kBAAkB,CAAC;AAChG,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAoB3D,6GAA6G;AAC7G,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;AAErD,4GAA4G;AAC5G,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5F,8FAA8F;AAC9F,SAAS,SAAS;IAChB,OAAO,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC;AACtC,CAAC;AAED,uDAAuD;AACvD,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAQ,KAA2C,EAAE,IAAI,IAAI,EAAE,CAAC;AAClE,CAAC;AAED,wDAAwD;AACxD,SAAS,SAAS,CAAC,KAAc;IAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC3B,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,SAAS,CAAC;AACjD,CAAC;AAED,0GAA0G;AAC1G,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,SAAS,CAAC,KAAK,CAAC;YAAE,OAAO,QAAQ,CAAC;QACtC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,sFAAsF;AACtF,KAAK,UAAU,MAAM,CAAC,MAAc;IAClC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,SAAS,CAAC,KAAK,CAAC;YAAE,OAAO,SAAS,CAAC;QACvC,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,MAAc;IAChC,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzD,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AACjF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,EAAU,EAAE,OAAe;IACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IACtC,KAAK,IAAI,IAAI,GAAG,EAAE,GAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,QAAQ;gBAAE,MAAM,KAAK,CAAC;YAC7E,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAED,kHAAkH;AAClH,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,OAAO;YAAE,OAAO;QACtC,4FAA4F;QAC5F,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACzB,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YACP,sFAAsF;QACxF,CAAC;IACH,CAAC;AACH,CAAC;AAED,uGAAuG;AACvG,KAAK,UAAU,UAAU,CAAC,MAAc;IACtC,IAAI,SAAS,EAAE;QAAE,OAAO;IACxB,IAAI,MAA8B,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACjC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAAE,MAAM,KAAK,CAAC;IACtD,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,EAAE,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAY,EACZ,IAAyB,EACzB,OAAO,GAA2B,EAAE;IAEpC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,MAA8B,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,IAAI,KAAK,CAAC,CAAC;QAC/C,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE,CAAC,CAAC;QACvE,4EAA4E;QAC5E,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,KAAK;YAAE,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,MAAM,GAAG,SAAS,CAAC;QACnB,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,2DAA2D;YAC7D,CAAC;QACH,CAAC;QACD,IAAI,OAAO;YAAE,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,KAAK,CAAC;IACd,CAAC;IACD,IAAI,KAAK;QAAE,MAAM,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;AAC/C,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The edit of a file, one at a time for each path (roadmap O51). An edit reads the file, changes
|
|
3
|
+
* the text and writes it with `writeFileAtomic()`, while it holds the lock of the path.
|
|
4
|
+
*/
|
|
5
|
+
import { type WriteFileAtomicOptions } from "./atomic.js";
|
|
6
|
+
/** The options of `editFile()`. */
|
|
7
|
+
export interface EditFileOptions extends WriteFileAtomicOptions {
|
|
8
|
+
/** The text that the transform gets when the file does not exist. Without it, the transform gets `undefined`. */
|
|
9
|
+
initial?: string | undefined;
|
|
10
|
+
/** The abort of this signal stops the wait for the lock of the path. */
|
|
11
|
+
signal?: AbortSignal | undefined;
|
|
12
|
+
}
|
|
13
|
+
/** The result of `editFile()`. */
|
|
14
|
+
export interface EditFileResult {
|
|
15
|
+
/** True when the function wrote the file. */
|
|
16
|
+
readonly changed: boolean;
|
|
17
|
+
/** The text of the file after the edit, or `undefined` when the file does not exist. */
|
|
18
|
+
readonly content: string | undefined;
|
|
19
|
+
}
|
|
20
|
+
/** The function that changes the text. It gives the new text, or `undefined` to write nothing. */
|
|
21
|
+
export type EditTransform<Current> = (current: Current) => string | undefined | PromiseLike<string | undefined>;
|
|
22
|
+
/**
|
|
23
|
+
* This function changes a file with `transform`, one edit at a time for each path in the process.
|
|
24
|
+
*
|
|
25
|
+
* - The function reads the file. A missing file gives `options.initial`, or `undefined`.
|
|
26
|
+
* - Then it waits for `transform` and writes its text with `writeFileAtomic()`.
|
|
27
|
+
* - A transform that gives the same text, or `undefined`, writes nothing.
|
|
28
|
+
* - Two paths of the same file share the lock, also when the case of the letters is not the same on Windows.
|
|
29
|
+
*
|
|
30
|
+
* The lock is in the process. It does not stop another process. The options of `writeFileAtomic()` apply to the write.
|
|
31
|
+
*/
|
|
32
|
+
export declare function editFile(path: string, transform: EditTransform<string>, options: EditFileOptions & {
|
|
33
|
+
initial: string;
|
|
34
|
+
}): Promise<EditFileResult>;
|
|
35
|
+
export declare function editFile(path: string, transform: EditTransform<string | undefined>, options?: EditFileOptions): Promise<EditFileResult>;
|
|
36
|
+
//# sourceMappingURL=edit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"edit.d.ts","sourceRoot":"","sources":["../../src/files/edit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAmB,KAAK,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAG3E,mCAAmC;AACnC,MAAM,WAAW,eAAgB,SAAQ,sBAAsB;IAC7D,iHAAiH;IACjH,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,wEAAwE;IACxE,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CAClC;AAED,kCAAkC;AAClC,MAAM,WAAW,cAAc;IAC7B,6CAA6C;IAC7C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,wFAAwF;IACxF,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC;AAED,kGAAkG;AAClG,MAAM,MAAM,aAAa,CAAC,OAAO,IAAI,CAAC,OAAO,EAAE,OAAO,KAAK,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;AA6BhH;;;;;;;;;GASG;AACH,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,EAChC,OAAO,EAAE,eAAe,GAAG;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GAC7C,OAAO,CAAC,cAAc,CAAC,CAAC;AAC3B,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,aAAa,CAAC,MAAM,GAAG,SAAS,CAAC,EAC5C,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,CAAC,CAAC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The edit of a file, one at a time for each path (roadmap O51). An edit reads the file, changes
|
|
3
|
+
* the text and writes it with `writeFileAtomic()`, while it holds the lock of the path.
|
|
4
|
+
*/
|
|
5
|
+
import { readFile, realpath } from "node:fs/promises";
|
|
6
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
7
|
+
import { writeFileAtomic } from "./atomic.js";
|
|
8
|
+
import { KeyedLock } from "./lock.js";
|
|
9
|
+
let pathLock;
|
|
10
|
+
/**
|
|
11
|
+
* The key of the lock of a path. The key is absolute, with the real path of the folder. On Windows,
|
|
12
|
+
* the case of the letters is not significant. The function does not follow a symlink of the file.
|
|
13
|
+
*/
|
|
14
|
+
async function lockKeyOf(absolute) {
|
|
15
|
+
let folder = dirname(absolute);
|
|
16
|
+
try {
|
|
17
|
+
folder = await realpath(folder);
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
// A folder that does not exist keeps its path. The write then fails.
|
|
21
|
+
}
|
|
22
|
+
const key = join(folder, basename(absolute));
|
|
23
|
+
return process.platform === "win32" ? key.toLowerCase() : key;
|
|
24
|
+
}
|
|
25
|
+
/** The text of the file, or `undefined` when the file does not exist. Another error is an error. */
|
|
26
|
+
async function readOrUndefined(path, encoding) {
|
|
27
|
+
try {
|
|
28
|
+
return await readFile(path, { encoding });
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
if (error.code === "ENOENT")
|
|
32
|
+
return undefined;
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export async function editFile(path, transform, options = {}) {
|
|
37
|
+
// The overloads give a transform of `string` only with `initial`, so the current text is a string then
|
|
38
|
+
const change = transform;
|
|
39
|
+
const { initial, signal, ...write } = options;
|
|
40
|
+
const target = resolve(path);
|
|
41
|
+
pathLock ??= new KeyedLock();
|
|
42
|
+
const release = await pathLock.acquire(await lockKeyOf(target), { signal });
|
|
43
|
+
try {
|
|
44
|
+
const encoding = write.encoding ?? "utf8";
|
|
45
|
+
const current = await readOrUndefined(target, encoding);
|
|
46
|
+
const next = await change(current ?? initial);
|
|
47
|
+
if (next === undefined || next === current)
|
|
48
|
+
return { changed: false, content: current };
|
|
49
|
+
await writeFileAtomic(target, next, { ...write, encoding });
|
|
50
|
+
return { changed: true, content: next };
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
release();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=edit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"edit.js","sourceRoot":"","sources":["../../src/files/edit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,eAAe,EAA+B,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAqBtC,IAAI,QAA+B,CAAC;AAEpC;;;GAGG;AACH,KAAK,UAAU,SAAS,CAAC,QAAgB;IACvC,IAAI,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,qEAAqE;IACvE,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC7C,OAAO,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;AAChE,CAAC;AAED,oGAAoG;AACpG,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,QAAwB;IACnE,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QACzE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAsBD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAY,EACZ,SAAoE,EACpE,OAAO,GAAoB,EAAE;IAE7B,uGAAuG;IACvG,MAAM,MAAM,GAAG,SAA8C,CAAC;IAC9D,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,QAAQ,KAAK,IAAI,SAAS,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC;QAC9C,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACxF,MAAM,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;YAAS,CAAC;QACT,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The file helpers of the package (roadmap O51): the `./files` entry. They are functions, not
|
|
3
|
+
* procedures, and an import of this entry registers nothing.
|
|
4
|
+
*
|
|
5
|
+
* - `writeFileAtomic()` writes a file in one step for each reader.
|
|
6
|
+
* - `healNdjsonTail()` repairs the end of an append-only NDJSON file after a crash.
|
|
7
|
+
* - `KeyedLock` and `withLock()` let one caller at a time hold a key.
|
|
8
|
+
* - `editFile()` changes a file, one edit at a time for each path.
|
|
9
|
+
*/
|
|
10
|
+
export { writeFileAtomic, type WriteFileAtomicOptions } from "./atomic.js";
|
|
11
|
+
export { healNdjsonTail, type HealNdjsonTailOptions, type NdjsonTailAction, type NdjsonTailResult } from "./ndjson-tail.js";
|
|
12
|
+
export { KeyedLock, withLock, type LockOutcome, type LockRelease, type LockSkipOptions, type LockWaitOptions, } from "./lock.js";
|
|
13
|
+
export { editFile, type EditFileOptions, type EditFileResult, type EditTransform } from "./edit.js";
|
|
14
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/files/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,KAAK,qBAAqB,EAAE,KAAK,gBAAgB,EAAE,KAAK,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAC5H,OAAO,EACL,SAAS,EACT,QAAQ,EACR,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,KAAK,eAAe,GACrB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,QAAQ,EAAE,KAAK,eAAe,EAAE,KAAK,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The file helpers of the package (roadmap O51): the `./files` entry. They are functions, not
|
|
3
|
+
* procedures, and an import of this entry registers nothing.
|
|
4
|
+
*
|
|
5
|
+
* - `writeFileAtomic()` writes a file in one step for each reader.
|
|
6
|
+
* - `healNdjsonTail()` repairs the end of an append-only NDJSON file after a crash.
|
|
7
|
+
* - `KeyedLock` and `withLock()` let one caller at a time hold a key.
|
|
8
|
+
* - `editFile()` changes a file, one edit at a time for each path.
|
|
9
|
+
*/
|
|
10
|
+
export { writeFileAtomic } from "./atomic.js";
|
|
11
|
+
export { healNdjsonTail } from "./ndjson-tail.js";
|
|
12
|
+
export { KeyedLock, withLock, } from "./lock.js";
|
|
13
|
+
export { editFile } from "./edit.js";
|
|
14
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/files/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,eAAe,EAA+B,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,cAAc,EAA4E,MAAM,kBAAkB,CAAC;AAC5H,OAAO,EACL,SAAS,EACT,QAAQ,GAKT,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,QAAQ,EAAiE,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A lock in the process for each key (roadmap O51). One holder at a time has a key. The other
|
|
3
|
+
* callers wait in the order of their calls, or they skip their work.
|
|
4
|
+
*/
|
|
5
|
+
/** The function that gives the lock back. A second call does nothing. */
|
|
6
|
+
export type LockRelease = () => void;
|
|
7
|
+
/** The options of a call that waits when the lock has a holder. */
|
|
8
|
+
export interface LockWaitOptions {
|
|
9
|
+
/** With `wait` (the default), the call waits for the lock. */
|
|
10
|
+
ifHeld?: "wait" | undefined;
|
|
11
|
+
/** The abort of this signal stops the wait. The call then rejects with the reason of the signal. */
|
|
12
|
+
signal?: AbortSignal | undefined;
|
|
13
|
+
}
|
|
14
|
+
/** The options of a call that skips its work when the lock has a holder. */
|
|
15
|
+
export interface LockSkipOptions {
|
|
16
|
+
/** With `skip`, the call does not start the function when the lock has a holder. */
|
|
17
|
+
ifHeld: "skip";
|
|
18
|
+
}
|
|
19
|
+
/** The result of a call with `ifHeld: "skip"`: the value of the function, or `ran: false` when the lock had a holder. */
|
|
20
|
+
export type LockOutcome<T> = {
|
|
21
|
+
readonly ran: true;
|
|
22
|
+
readonly value: T;
|
|
23
|
+
} | {
|
|
24
|
+
readonly ran: false;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* A set of locks, one for each key, in one process. Each instance has its own keys.
|
|
28
|
+
*
|
|
29
|
+
* - One holder at a time has a key. The waiters get the key in the order of their calls.
|
|
30
|
+
* - The lock is not reentrant. A holder that asks for its own key again waits for ever.
|
|
31
|
+
* - A key without a holder uses no memory.
|
|
32
|
+
*/
|
|
33
|
+
export declare class KeyedLock {
|
|
34
|
+
/** The waiters of each key that has a holder. A key without a holder is not in the map. */
|
|
35
|
+
private readonly queues;
|
|
36
|
+
/** The number of keys that have a holder. */
|
|
37
|
+
get size(): number;
|
|
38
|
+
/** This method tells if the key has a holder. */
|
|
39
|
+
isHeld(key: string): boolean;
|
|
40
|
+
/** This method takes the key when it has no holder. Otherwise it gives `undefined` and does not wait. */
|
|
41
|
+
tryAcquire(key: string): LockRelease | undefined;
|
|
42
|
+
/**
|
|
43
|
+
* This method gives the key to the caller. When the key has a holder, the caller waits after the
|
|
44
|
+
* other waiters. The abort of `options.signal` stops the wait.
|
|
45
|
+
*/
|
|
46
|
+
acquire(key: string, options?: {
|
|
47
|
+
signal?: AbortSignal | undefined;
|
|
48
|
+
}): Promise<LockRelease>;
|
|
49
|
+
/**
|
|
50
|
+
* This method starts `fn` with the key, and gives the key back when `fn` ends or throws.
|
|
51
|
+
* With `ifHeld: "skip"`, it does not start `fn` when the key has a holder.
|
|
52
|
+
*/
|
|
53
|
+
run<T>(key: string, fn: () => T | PromiseLike<T>, options?: LockWaitOptions): Promise<T>;
|
|
54
|
+
run<T>(key: string, fn: () => T | PromiseLike<T>, options: LockSkipOptions): Promise<LockOutcome<T>>;
|
|
55
|
+
/** The start of a turn of a key without a holder: an empty queue in the map. */
|
|
56
|
+
private take;
|
|
57
|
+
/**
|
|
58
|
+
* The release of one turn of the key. It gives the key to the first waiter of `queue`, or removes
|
|
59
|
+
* the key. The queue stays in the map while the key has a holder.
|
|
60
|
+
*/
|
|
61
|
+
private releaseOf;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* This function starts `fn` with the key of the lock of the process, and gives the key back when
|
|
65
|
+
* `fn` ends or throws. The callers of one key run one at a time, in the order of their calls.
|
|
66
|
+
*
|
|
67
|
+
* - With `ifHeld: "skip"`, the function does not start `fn` when the key has a holder. It gives `{ ran: false }`.
|
|
68
|
+
* - All the callers in the process share the keys. For keys of your own, make a `KeyedLock`.
|
|
69
|
+
*/
|
|
70
|
+
export declare function withLock<T>(key: string, fn: () => T | PromiseLike<T>, options?: LockWaitOptions): Promise<T>;
|
|
71
|
+
export declare function withLock<T>(key: string, fn: () => T | PromiseLike<T>, options: LockSkipOptions): Promise<LockOutcome<T>>;
|
|
72
|
+
//# sourceMappingURL=lock.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["../../src/files/lock.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,yEAAyE;AACzE,MAAM,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC;AAErC,mEAAmE;AACnE,MAAM,WAAW,eAAe;IAC9B,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,oGAAoG;IACpG,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CAClC;AAED,4EAA4E;AAC5E,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,yHAAyH;AACzH,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI;IAAE,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAAG;IAAE,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAA;CAAE,CAAC;AAYjG;;;;;;GAMG;AACH,qBAAa,SAAS;IACpB,2FAA2F;IAC3F,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA+B;IAEtD,6CAA6C;IAC7C,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,iDAAiD;IACjD,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAE3B;IAED,yGAAyG;IACzG,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAE/C;IAED;;;OAGG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;KAAO,GAAG,OAAO,CAAC,WAAW,CAAC,CAkB7F;IAED;;;OAGG;IACH,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzF,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAuBrG,gFAAgF;IAChF,OAAO,CAAC,IAAI;IAMZ;;;OAGG;IACH,OAAO,CAAC,SAAS;CAclB;AAID;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAC9G,wBAAgB,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A lock in the process for each key (roadmap O51). One holder at a time has a key. The other
|
|
3
|
+
* callers wait in the order of their calls, or they skip their work.
|
|
4
|
+
*/
|
|
5
|
+
/** The function that does nothing: the `detach` of a waiter without a signal. */
|
|
6
|
+
function noop() { }
|
|
7
|
+
/**
|
|
8
|
+
* A set of locks, one for each key, in one process. Each instance has its own keys.
|
|
9
|
+
*
|
|
10
|
+
* - One holder at a time has a key. The waiters get the key in the order of their calls.
|
|
11
|
+
* - The lock is not reentrant. A holder that asks for its own key again waits for ever.
|
|
12
|
+
* - A key without a holder uses no memory.
|
|
13
|
+
*/
|
|
14
|
+
export class KeyedLock {
|
|
15
|
+
/** The waiters of each key that has a holder. A key without a holder is not in the map. */
|
|
16
|
+
queues = new Map();
|
|
17
|
+
/** The number of keys that have a holder. */
|
|
18
|
+
get size() {
|
|
19
|
+
return this.queues.size;
|
|
20
|
+
}
|
|
21
|
+
/** This method tells if the key has a holder. */
|
|
22
|
+
isHeld(key) {
|
|
23
|
+
return this.queues.has(key);
|
|
24
|
+
}
|
|
25
|
+
/** This method takes the key when it has no holder. Otherwise it gives `undefined` and does not wait. */
|
|
26
|
+
tryAcquire(key) {
|
|
27
|
+
return this.queues.has(key) ? undefined : this.take(key);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* This method gives the key to the caller. When the key has a holder, the caller waits after the
|
|
31
|
+
* other waiters. The abort of `options.signal` stops the wait.
|
|
32
|
+
*/
|
|
33
|
+
acquire(key, options = {}) {
|
|
34
|
+
const { signal } = options;
|
|
35
|
+
if (signal?.aborted)
|
|
36
|
+
return Promise.reject(signal.reason);
|
|
37
|
+
const queue = this.queues.get(key);
|
|
38
|
+
if (!queue)
|
|
39
|
+
return Promise.resolve(this.take(key));
|
|
40
|
+
return new Promise((resolve, reject) => {
|
|
41
|
+
const waiter = { grant: resolve, detach: noop };
|
|
42
|
+
queue.push(waiter);
|
|
43
|
+
if (signal) {
|
|
44
|
+
const onAbort = () => {
|
|
45
|
+
// The grant removes this listener, so the waiter is still in the queue here
|
|
46
|
+
queue.splice(queue.indexOf(waiter), 1);
|
|
47
|
+
reject(signal.reason);
|
|
48
|
+
};
|
|
49
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
50
|
+
waiter.detach = () => signal.removeEventListener("abort", onAbort);
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
async run(key, fn, options = {}) {
|
|
55
|
+
if (options.ifHeld === "skip") {
|
|
56
|
+
const release = this.tryAcquire(key);
|
|
57
|
+
if (!release)
|
|
58
|
+
return { ran: false };
|
|
59
|
+
try {
|
|
60
|
+
return { ran: true, value: await fn() };
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
release();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
const release = await this.acquire(key, { signal: options.signal });
|
|
67
|
+
try {
|
|
68
|
+
return await fn();
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
release();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/** The start of a turn of a key without a holder: an empty queue in the map. */
|
|
75
|
+
take(key) {
|
|
76
|
+
const queue = [];
|
|
77
|
+
this.queues.set(key, queue);
|
|
78
|
+
return this.releaseOf(key, queue);
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The release of one turn of the key. It gives the key to the first waiter of `queue`, or removes
|
|
82
|
+
* the key. The queue stays in the map while the key has a holder.
|
|
83
|
+
*/
|
|
84
|
+
releaseOf(key, queue) {
|
|
85
|
+
let released = false;
|
|
86
|
+
return () => {
|
|
87
|
+
if (released)
|
|
88
|
+
return;
|
|
89
|
+
released = true;
|
|
90
|
+
const next = queue.shift();
|
|
91
|
+
if (next) {
|
|
92
|
+
next.detach();
|
|
93
|
+
next.grant(this.releaseOf(key, queue));
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
this.queues.delete(key);
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
let shared;
|
|
102
|
+
export function withLock(key, fn, options) {
|
|
103
|
+
shared ??= new KeyedLock();
|
|
104
|
+
return options?.ifHeld === "skip" ? shared.run(key, fn, options) : shared.run(key, fn, options);
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=lock.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lock.js","sourceRoot":"","sources":["../../src/files/lock.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA6BH,iFAAiF;AACjF,SAAS,IAAI,KAAU,CAAC;AAExB;;;;;;GAMG;AACH,MAAM,OAAO,SAAS;IACpB,2FAA2F;IAC1E,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEtD,6CAA6C;IAC7C,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1B,CAAC;IAED,iDAAiD;IACjD,MAAM,CAAC,GAAW;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAED,yGAAyG;IACzG,UAAU,CAAC,GAAW;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,GAAW,EAAE,OAAO,GAAyC,EAAE;QACrE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAC3B,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,OAAO,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAClD,MAAM,MAAM,GAAW,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACxD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACnB,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,OAAO,GAAG,GAAS,EAAE;oBACzB,4EAA4E;oBAC5E,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACxB,CAAC,CAAC;gBACF,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACrE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAQD,KAAK,CAAC,GAAG,CACP,GAAW,EACX,EAA4B,EAC5B,OAAO,GAAsC,EAAE;QAE/C,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,CAAC,OAAO;gBAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YAC1C,CAAC;oBAAS,CAAC;gBACT,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACpE,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,gFAAgF;IACxE,IAAI,CAAC,GAAW;QACtB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACK,SAAS,CAAC,GAAW,EAAE,KAAe;QAC5C,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,OAAO,GAAG,EAAE;YACV,IAAI,QAAQ;gBAAE,OAAO;YACrB,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC;IACJ,CAAC;CACF;AAED,IAAI,MAA6B,CAAC;AAWlC,MAAM,UAAU,QAAQ,CACtB,GAAW,EACX,EAA4B,EAC5B,OAA2C;IAE3C,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;IAC3B,OAAO,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;AAClG,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The repair of the end of an append-only NDJSON file after a crash (roadmap O51).
|
|
3
|
+
*
|
|
4
|
+
* A crash during an append can leave a part of a line at the end of the file. The next append
|
|
5
|
+
* then puts its line after that part, and the file has a bad line in its middle.
|
|
6
|
+
* `healNdjsonTail()` repairs the end before the next append.
|
|
7
|
+
*/
|
|
8
|
+
/** What `healNdjsonTail()` did: nothing, a cut of a partial last line, or a newline after a complete last record. */
|
|
9
|
+
export type NdjsonTailAction = "none" | "truncated" | "newline";
|
|
10
|
+
/** The result of `healNdjsonTail()`. */
|
|
11
|
+
export interface NdjsonTailResult {
|
|
12
|
+
/** The change to the file. */
|
|
13
|
+
readonly action: NdjsonTailAction;
|
|
14
|
+
/** The number of bytes that the function removed or added. It is 0 for `none`. */
|
|
15
|
+
readonly bytes: number;
|
|
16
|
+
/** The size of the file after the repair. It is 0 for a file that does not exist. */
|
|
17
|
+
readonly size: number;
|
|
18
|
+
}
|
|
19
|
+
/** The options of `healNdjsonTail()`. */
|
|
20
|
+
export interface HealNdjsonTailOptions {
|
|
21
|
+
/**
|
|
22
|
+
* The maximum length in bytes of the last line. The function reads no more than this number of
|
|
23
|
+
* bytes from the end. A longer last line is an error. The default is 16 MiB.
|
|
24
|
+
*/
|
|
25
|
+
maxLineBytes?: number | undefined;
|
|
26
|
+
/** With `false`, the function does not flush a change to the disk. The default is `true`. */
|
|
27
|
+
fsync?: boolean | undefined;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* This function repairs the end of an append-only NDJSON file after a crash. Then the file ends in a newline, or it is empty.
|
|
31
|
+
*
|
|
32
|
+
* - A missing file, an empty file and a file that ends in a newline get no change.
|
|
33
|
+
* - A last line that is not valid JSON is a part of a record. The function cuts it off with `ftruncate`.
|
|
34
|
+
* - A last line that is valid JSON is a complete record without its newline. The function adds the newline.
|
|
35
|
+
* - The function reads only the last line, from the end of the file. It does not read the whole file.
|
|
36
|
+
*
|
|
37
|
+
* The records must be objects or arrays. A part of a number or of a literal can be valid JSON too.
|
|
38
|
+
* The function does not lock the file. Start it before an append, in the same lock as the append.
|
|
39
|
+
*/
|
|
40
|
+
export declare function healNdjsonTail(path: string, options?: HealNdjsonTailOptions): Promise<NdjsonTailResult>;
|
|
41
|
+
//# sourceMappingURL=ndjson-tail.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ndjson-tail.d.ts","sourceRoot":"","sources":["../../src/files/ndjson-tail.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,qHAAqH;AACrH,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,WAAW,GAAG,SAAS,CAAC;AAEhE,wCAAwC;AACxC,MAAM,WAAW,gBAAgB;IAC/B,8BAA8B;IAC9B,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,kFAAkF;IAClF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qFAAqF;IACrF,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,yCAAyC;AACzC,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,6FAA6F;IAC7F,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAC7B;AAgCD;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAgDjH"}
|