@hraness/sponge 0.2.6 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -9
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +38 -16
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +9 -46
- package/dist/src/digest/editorial.js +57 -49
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +49 -28
- package/dist/src/digest/pipeline.js +369 -102
- package/dist/src/digest/schema.d.ts +11 -25
- package/dist/src/digest/schema.js +86 -18
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +3 -6
- package/dist/src/importers/jungle-digest.js +196 -69
- package/dist/src/importers/jungle-store.d.ts +19 -6
- package/dist/src/importers/jungle-store.js +124 -28
- package/dist/src/importers/jungle.d.ts +14 -14
- package/dist/src/importers/jungle.js +142 -31
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +106 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -79
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -1,14 +1,29 @@
|
|
|
1
1
|
import { lstat, link, unlink, readdir, rm, realpath } from 'node:fs/promises';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { randomUUID } from 'node:crypto';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { directory, identifier, readJournal, writeJournal } from './journal.js';
|
|
5
|
+
import { syncDirectoryAsync } from '../util/fsync.js';
|
|
6
6
|
import { processIdentity, identityAlive } from './identity.js';
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
async function
|
|
11
|
-
|
|
7
|
+
import { acquireIdentityLock } from './lock.js';
|
|
8
|
+
import { SpongeError, codeOf } from '../errors.js';
|
|
9
|
+
const pending = () => new SpongeError('cleanup-required');
|
|
10
|
+
async function ownerRecord(path) {
|
|
11
|
+
const record = await readJournal(path);
|
|
12
|
+
if (record.schemaVersion !== 1 ||
|
|
13
|
+
typeof record.incarnation !== 'string' ||
|
|
14
|
+
!record.identity ||
|
|
15
|
+
!Number.isSafeInteger(record.identity.pid) ||
|
|
16
|
+
record.identity.uid !== process.getuid?.() ||
|
|
17
|
+
typeof record.identity.started !== 'string')
|
|
18
|
+
throw pending();
|
|
19
|
+
identifier(record.incarnation);
|
|
20
|
+
return record;
|
|
21
|
+
}
|
|
22
|
+
async function sameFile(a, b) {
|
|
23
|
+
const [x, y] = await Promise.all([lstat(a), lstat(b)]);
|
|
24
|
+
if (!x.isFile() || !y.isFile() || x.ino !== y.ino || x.dev !== y.dev)
|
|
25
|
+
throw pending();
|
|
26
|
+
}
|
|
12
27
|
export async function acquireWorkerCustody(root) {
|
|
13
28
|
const workers = await directory(join(root, 'runtime', 'workers')), incarnation = randomUUID(), ownerDirectory = await directory(join(workers, incarnation));
|
|
14
29
|
const identity = await processIdentity(process.pid);
|
|
@@ -18,14 +33,22 @@ export async function acquireWorkerCustody(root) {
|
|
|
18
33
|
await writeJournal(file, { schemaVersion: 1, incarnation, identity });
|
|
19
34
|
try {
|
|
20
35
|
await link(file, lock);
|
|
21
|
-
await
|
|
36
|
+
await syncDirectoryAsync(join(root, 'runtime'));
|
|
22
37
|
}
|
|
23
38
|
catch (error) {
|
|
24
39
|
if (error.code === 'EEXIST')
|
|
25
|
-
throw new
|
|
40
|
+
throw new SpongeError('owner-busy');
|
|
26
41
|
throw error;
|
|
27
42
|
}
|
|
28
|
-
return {
|
|
43
|
+
return {
|
|
44
|
+
incarnation,
|
|
45
|
+
directory: ownerDirectory,
|
|
46
|
+
release: async () => {
|
|
47
|
+
await sameFile(file, lock);
|
|
48
|
+
await unlink(lock);
|
|
49
|
+
await syncDirectoryAsync(join(root, 'runtime'));
|
|
50
|
+
},
|
|
51
|
+
};
|
|
29
52
|
}
|
|
30
53
|
/** Receipts are specific to every pre-launch ticket. Missing receipts never mean absent children. */
|
|
31
54
|
export async function proveAttemptCleanup(ownerDirectory, incarnation, attemptId) {
|
|
@@ -34,23 +57,34 @@ export async function proveAttemptCleanup(ownerDirectory, incarnation, attemptId
|
|
|
34
57
|
const root = join(ownerDirectory, 'attempts', attemptId);
|
|
35
58
|
try {
|
|
36
59
|
const stat = await lstat(root);
|
|
37
|
-
if (!stat.isDirectory() ||
|
|
60
|
+
if (!stat.isDirectory() ||
|
|
61
|
+
stat.isSymbolicLink() ||
|
|
62
|
+
(await realpath(root)) !== root ||
|
|
63
|
+
stat.uid !== process.getuid?.() ||
|
|
64
|
+
(stat.mode & 0o077) !== 0)
|
|
38
65
|
return false;
|
|
39
66
|
entries = await readdir(root);
|
|
40
67
|
}
|
|
41
68
|
catch {
|
|
42
69
|
return false;
|
|
43
70
|
}
|
|
44
|
-
for (const entry of entries.filter(name => /\.(settled|supervisor)\.json$/.test(name)))
|
|
71
|
+
for (const entry of entries.filter((name) => /\.(settled|supervisor)\.json$/.test(name)))
|
|
45
72
|
if (!entries.includes(entry.replace(/\.(settled|supervisor)\.json$/, '.pending.json')))
|
|
46
73
|
return false;
|
|
47
|
-
for (const entry of entries.filter(name => name.endsWith('.pending.json'))) {
|
|
74
|
+
for (const entry of entries.filter((name) => name.endsWith('.pending.json'))) {
|
|
48
75
|
try {
|
|
49
76
|
const ticket = await readJournal(join(root, entry)), receipt = await readJournal(join(root, entry.replace('.pending.', '.settled.')));
|
|
50
77
|
if (typeof ticket.ticket !== 'string' || entry !== ticket.ticket + '.pending.json')
|
|
51
78
|
return false;
|
|
52
79
|
identifier(ticket.ticket);
|
|
53
|
-
if (ticket.schemaVersion !== 1 ||
|
|
80
|
+
if (ticket.schemaVersion !== 1 ||
|
|
81
|
+
ticket.attemptId !== attemptId ||
|
|
82
|
+
ticket.ownerIncarnation !== incarnation ||
|
|
83
|
+
receipt.schemaVersion !== 1 ||
|
|
84
|
+
receipt.ticket !== ticket.ticket ||
|
|
85
|
+
receipt.attemptId !== attemptId ||
|
|
86
|
+
receipt.ownerIncarnation !== incarnation ||
|
|
87
|
+
receipt.cleanup !== 'settled')
|
|
54
88
|
return false;
|
|
55
89
|
}
|
|
56
90
|
catch {
|
|
@@ -59,7 +93,30 @@ export async function proveAttemptCleanup(ownerDirectory, incarnation, attemptId
|
|
|
59
93
|
}
|
|
60
94
|
return true;
|
|
61
95
|
}
|
|
62
|
-
|
|
96
|
+
/** Leaked journal temps (`.<uuid>.tmp`) in a dead owner's directory tree: its writer is gone, so nothing will rename them. */
|
|
97
|
+
async function sweepTemps(path, depth = 2) {
|
|
98
|
+
let entries;
|
|
99
|
+
try {
|
|
100
|
+
entries = await readdir(path, { withFileTypes: true });
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
if (error.code === 'ENOENT')
|
|
104
|
+
return;
|
|
105
|
+
throw error;
|
|
106
|
+
}
|
|
107
|
+
let removed = false;
|
|
108
|
+
for (const entry of entries) {
|
|
109
|
+
if (entry.isDirectory() && depth > 0)
|
|
110
|
+
await sweepTemps(join(path, entry.name), depth - 1);
|
|
111
|
+
else if (entry.isFile() && /^\.[0-9a-f-]{36}\.tmp$/.test(entry.name)) {
|
|
112
|
+
await unlink(join(path, entry.name));
|
|
113
|
+
removed = true;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (removed)
|
|
117
|
+
await syncDirectoryAsync(path);
|
|
118
|
+
}
|
|
119
|
+
export async function recoverWorker(store, options = {}) {
|
|
63
120
|
const lock = join(store.root, 'runtime', 'worker.lock');
|
|
64
121
|
let record;
|
|
65
122
|
try {
|
|
@@ -67,38 +124,41 @@ export async function recoverWorker(store) {
|
|
|
67
124
|
}
|
|
68
125
|
catch (error) {
|
|
69
126
|
if (error.code === 'ENOENT') {
|
|
70
|
-
if (store.ownerSnapshot())
|
|
127
|
+
if (store.queue.ownerSnapshot())
|
|
71
128
|
throw pending();
|
|
72
129
|
return { recovered: false, attempts: 0 };
|
|
73
130
|
}
|
|
74
131
|
throw pending();
|
|
75
132
|
}
|
|
76
133
|
if (await identityAlive(record.identity))
|
|
77
|
-
throw new
|
|
134
|
+
throw new SpongeError('owner-busy');
|
|
78
135
|
const ownerDirectory = join(store.root, 'runtime', 'workers', record.incarnation), file = join(ownerDirectory, 'owner.json');
|
|
79
136
|
await sameFile(file, lock);
|
|
80
|
-
// Serialize recovery.
|
|
81
|
-
|
|
137
|
+
// Serialize recovery. The guard names the recovering process; only exact proof that IT died lets a later recovery take over.
|
|
138
|
+
let guard;
|
|
82
139
|
try {
|
|
83
|
-
await
|
|
140
|
+
guard = await acquireIdentityLock(join(store.root, 'runtime', 'worker-recovery.lock'));
|
|
84
141
|
}
|
|
85
142
|
catch (error) {
|
|
86
|
-
if (error
|
|
143
|
+
if (codeOf(error) === 'owner-busy')
|
|
87
144
|
throw pending();
|
|
88
145
|
throw error;
|
|
89
146
|
}
|
|
90
147
|
try {
|
|
148
|
+
options.fault?.('guarded');
|
|
91
149
|
await sameFile(file, lock);
|
|
92
150
|
if (await identityAlive(record.identity))
|
|
93
|
-
throw new
|
|
94
|
-
|
|
151
|
+
throw new SpongeError('owner-busy');
|
|
152
|
+
// No owner row means the store side already committed (a recovery that died before unlinking the worker lock).
|
|
153
|
+
const owner = store.queue.ownerSnapshot();
|
|
95
154
|
if (owner && owner.ownerIncarnation !== record.incarnation)
|
|
96
155
|
throw pending();
|
|
97
|
-
const attempts = owner ? store.ownerAttempts(owner) : [];
|
|
156
|
+
const attempts = owner ? store.queue.ownerAttempts(owner) : [];
|
|
98
157
|
for (const attempt of attempts)
|
|
99
|
-
if (attempt.cleanup === 'pending' &&
|
|
158
|
+
if (attempt.cleanup === 'pending' &&
|
|
159
|
+
!(await proveAttemptCleanup(ownerDirectory, record.incarnation, attempt.attemptId)))
|
|
100
160
|
throw pending();
|
|
101
|
-
const attemptIds = new Set(attempts.map(a => a.attemptId)), transientIds = new Map();
|
|
161
|
+
const attemptIds = new Set(attempts.map((a) => a.attemptId)), transientIds = new Map();
|
|
102
162
|
let journals = [];
|
|
103
163
|
try {
|
|
104
164
|
journals = await readdir(join(ownerDirectory, 'attempts'));
|
|
@@ -123,11 +183,20 @@ export async function recoverWorker(store) {
|
|
|
123
183
|
throw pending();
|
|
124
184
|
}
|
|
125
185
|
}
|
|
126
|
-
if (!marker
|
|
127
|
-
|
|
128
|
-
|
|
186
|
+
if (!marker) {
|
|
187
|
+
if (owner)
|
|
188
|
+
throw pending();
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
if (marker.schemaVersion !== 1 ||
|
|
192
|
+
!['query', 'import-verification'].includes(marker.kind) ||
|
|
193
|
+
marker.attemptId !== id ||
|
|
194
|
+
marker.ownerIncarnation !== record.incarnation)
|
|
195
|
+
throw pending();
|
|
196
|
+
transientIds.set(id, marker.kind);
|
|
197
|
+
}
|
|
129
198
|
}
|
|
130
|
-
if (!await proveAttemptCleanup(ownerDirectory, record.incarnation, id))
|
|
199
|
+
if (!(await proveAttemptCleanup(ownerDirectory, record.incarnation, id)))
|
|
131
200
|
throw pending();
|
|
132
201
|
}
|
|
133
202
|
const staging = join(store.root, 'staging');
|
|
@@ -142,13 +211,17 @@ export async function recoverWorker(store) {
|
|
|
142
211
|
catch {
|
|
143
212
|
throw pending();
|
|
144
213
|
}
|
|
145
|
-
if (marker.schemaVersion !== 1 ||
|
|
214
|
+
if (marker.schemaVersion !== 1 ||
|
|
215
|
+
!['query', 'import-verification'].includes(marker.kind) ||
|
|
216
|
+
typeof marker.requestId !== 'string' ||
|
|
217
|
+
entry !== marker.kind + '-' + marker.requestId ||
|
|
218
|
+
typeof marker.ownerIncarnation !== 'string')
|
|
146
219
|
throw pending();
|
|
147
220
|
if (marker.ownerIncarnation === record.incarnation) {
|
|
148
221
|
if (transientIds.get(marker.requestId) !== marker.kind)
|
|
149
222
|
throw pending();
|
|
150
223
|
await rm(path, { recursive: true });
|
|
151
|
-
await
|
|
224
|
+
await syncDirectoryAsync(staging);
|
|
152
225
|
}
|
|
153
226
|
continue;
|
|
154
227
|
}
|
|
@@ -167,19 +240,19 @@ export async function recoverWorker(store) {
|
|
|
167
240
|
throw pending();
|
|
168
241
|
if (attemptIds.has(marker.requestId)) {
|
|
169
242
|
await rm(path, { recursive: true });
|
|
170
|
-
await
|
|
243
|
+
await syncDirectoryAsync(staging);
|
|
171
244
|
}
|
|
172
245
|
}
|
|
173
246
|
if (owner)
|
|
174
|
-
store.recoverOwner(owner, attempts.filter(a => a.cleanup === 'pending').map(a => a.attemptId));
|
|
247
|
+
store.queue.recoverOwner(owner, attempts.filter((a) => a.cleanup === 'pending').map((a) => a.attemptId));
|
|
248
|
+
options.fault?.('recovered');
|
|
249
|
+
await sweepTemps(ownerDirectory);
|
|
175
250
|
await sameFile(file, lock);
|
|
176
251
|
await unlink(lock);
|
|
177
|
-
await
|
|
252
|
+
await syncDirectoryAsync(join(store.root, 'runtime'));
|
|
178
253
|
return { recovered: true, attempts: attempts.length };
|
|
179
254
|
}
|
|
180
255
|
finally {
|
|
181
|
-
await
|
|
182
|
-
await unlink(recovery);
|
|
183
|
-
await syncDir(join(store.root, 'runtime'));
|
|
256
|
+
await guard.release();
|
|
184
257
|
}
|
|
185
258
|
}
|
|
@@ -17,7 +17,7 @@ export interface HandlerContext {
|
|
|
17
17
|
checkpoint: (stage: JobCheckpoint['stage']) => Promise<void>;
|
|
18
18
|
}
|
|
19
19
|
/** Handlers finish and settle storage only after all process and artifact cleanup is established. */
|
|
20
|
-
|
|
20
|
+
type JobHandler<T extends WorkerOutcome = WorkerOutcome> = (context: HandlerContext) => Promise<T>;
|
|
21
21
|
export interface QueueOptions<T extends WorkerOutcome> {
|
|
22
22
|
handlers: Readonly<Record<string, JobHandler<T>>>;
|
|
23
23
|
signal?: AbortSignal;
|
|
@@ -26,3 +26,4 @@ export interface QueueOptions<T extends WorkerOutcome> {
|
|
|
26
26
|
onCheckpoint?: (attempt: AttemptToken, stage: JobCheckpoint['stage']) => Promise<void>;
|
|
27
27
|
}
|
|
28
28
|
export declare function drainQueue<T extends WorkerOutcome>(store: LibraryStore, options: QueueOptions<T>): Promise<Array<T | WorkerFailure>>;
|
|
29
|
+
export {};
|
|
@@ -3,36 +3,46 @@ import { failure } from '../acquisition/errors.js';
|
|
|
3
3
|
import { acquireWorkerCustody, proveAttemptCleanup } from './custody.js';
|
|
4
4
|
import { supervisedRunner } from './supervisor.js';
|
|
5
5
|
import { writeJournal, directory } from './journal.js';
|
|
6
|
+
import { SpongeError } from '../errors.js';
|
|
6
7
|
export async function drainQueue(store, options) {
|
|
7
8
|
const custody = await acquireWorkerCustody(store.root);
|
|
8
9
|
let owner, settled = true;
|
|
9
10
|
const results = [];
|
|
10
11
|
try {
|
|
11
|
-
owner = store.acquireOwner(custody.incarnation);
|
|
12
|
+
owner = store.queue.acquireOwner(custody.incarnation);
|
|
12
13
|
while (!options.signal?.aborted) {
|
|
13
|
-
const attempt = options.jobId ? store.
|
|
14
|
+
const attempt = options.jobId ? store.queue.claim(owner, options.jobId) : store.queue.claimNext(owner);
|
|
14
15
|
if (!attempt)
|
|
15
16
|
break;
|
|
16
17
|
settled = false;
|
|
17
18
|
const attemptDirectory = await directory(join(custody.directory, 'attempts', attempt.attemptId));
|
|
18
|
-
const checkpoint = async (stage) => {
|
|
19
|
+
const checkpoint = async (stage) => {
|
|
20
|
+
store.queue.checkpoint(attempt, stage);
|
|
21
|
+
await writeJournal(join(attemptDirectory, 'checkpoint.json'), { schemaVersion: 1, ...attempt, stage });
|
|
22
|
+
await options.onCheckpoint?.(attempt, stage);
|
|
23
|
+
};
|
|
19
24
|
await checkpoint('acquiring');
|
|
20
|
-
const controller = new AbortController(), cancel = () => {
|
|
25
|
+
const controller = new AbortController(), cancel = () => {
|
|
26
|
+
store.queue.cancel(attempt.jobId);
|
|
27
|
+
controller.abort();
|
|
28
|
+
};
|
|
21
29
|
options.signal?.addEventListener('abort', cancel, { once: true });
|
|
22
|
-
const poll = setInterval(() => {
|
|
23
|
-
|
|
30
|
+
const poll = setInterval(() => {
|
|
31
|
+
try {
|
|
32
|
+
if (store.queue.cancelRequested(attempt.jobId))
|
|
33
|
+
controller.abort();
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
24
36
|
controller.abort();
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
controller.abort();
|
|
28
|
-
} }, 100);
|
|
37
|
+
}
|
|
38
|
+
}, 100);
|
|
29
39
|
try {
|
|
30
40
|
if (options.signal?.aborted)
|
|
31
41
|
cancel();
|
|
32
|
-
const source = store.
|
|
42
|
+
const source = store.queue.source(attempt.jobId), handler = source && options.handlers[source.kind];
|
|
33
43
|
if (!handler) {
|
|
34
|
-
store.
|
|
35
|
-
store.
|
|
44
|
+
store.queue.fail(attempt, 'unsupported');
|
|
45
|
+
store.queue.settle(attempt);
|
|
36
46
|
settled = true;
|
|
37
47
|
const result = { ...failure(attempt.attemptId, 'unsupported'), cleanup: 'settled' };
|
|
38
48
|
results.push(result);
|
|
@@ -41,17 +51,29 @@ export async function drainQueue(store, options) {
|
|
|
41
51
|
break;
|
|
42
52
|
continue;
|
|
43
53
|
}
|
|
44
|
-
const result = await handler({
|
|
54
|
+
const result = await handler({
|
|
55
|
+
store,
|
|
56
|
+
attempt,
|
|
57
|
+
signal: controller.signal,
|
|
58
|
+
runProcess: supervisedRunner(custody.directory, custody.incarnation, attempt.attemptId),
|
|
59
|
+
checkpoint,
|
|
60
|
+
});
|
|
45
61
|
results.push(result);
|
|
46
62
|
options.onResult?.(result);
|
|
47
|
-
const state = store.ownerAttempts(owner).find(a => a.attemptId === attempt.attemptId);
|
|
48
|
-
settled =
|
|
63
|
+
const state = store.queue.ownerAttempts(owner).find((a) => a.attemptId === attempt.attemptId);
|
|
64
|
+
settled =
|
|
65
|
+
result.cleanup === 'settled' &&
|
|
66
|
+
state?.cleanup === 'settled' &&
|
|
67
|
+
(await proveAttemptCleanup(custody.directory, custody.incarnation, attempt.attemptId));
|
|
49
68
|
if (!settled || options.jobId)
|
|
50
69
|
break;
|
|
51
70
|
if (!result.ok && result.retryable && ['unavailable', 'rate-limited'].includes(result.code)) {
|
|
52
|
-
const job = store.
|
|
71
|
+
const job = store.queue.list().find((j) => j.id === attempt.jobId), count = Number(job?.attempt_count ?? 3);
|
|
53
72
|
if (count < 3)
|
|
54
|
-
store.
|
|
73
|
+
store.queue.retry(attempt.jobId, {
|
|
74
|
+
automatic: true,
|
|
75
|
+
notBefore: new Date(Date.now() + Math.min(60000, 1000 * 2 ** count)).toISOString(),
|
|
76
|
+
});
|
|
55
77
|
}
|
|
56
78
|
}
|
|
57
79
|
finally {
|
|
@@ -60,13 +82,13 @@ export async function drainQueue(store, options) {
|
|
|
60
82
|
}
|
|
61
83
|
}
|
|
62
84
|
if (!settled)
|
|
63
|
-
throw
|
|
85
|
+
throw new SpongeError('cleanup-required');
|
|
64
86
|
return results;
|
|
65
87
|
}
|
|
66
88
|
finally {
|
|
67
89
|
if (owner && settled) {
|
|
68
90
|
try {
|
|
69
|
-
store.releaseOwner(owner);
|
|
91
|
+
store.queue.releaseOwner(owner);
|
|
70
92
|
}
|
|
71
93
|
catch {
|
|
72
94
|
settled = false;
|
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import { execFile } from 'node:child_process';
|
|
2
2
|
import { promisify } from 'node:util';
|
|
3
|
+
import { SpongeError } from '../errors.js';
|
|
3
4
|
const exec = promisify(execFile);
|
|
4
5
|
export async function processIdentity(pid) {
|
|
5
6
|
if (!Number.isSafeInteger(pid) || pid <= 0)
|
|
6
|
-
throw
|
|
7
|
+
throw new SpongeError('invalid-input');
|
|
7
8
|
try {
|
|
8
|
-
const { stdout } = await exec('/bin/ps', ['-p', String(pid), '-o', 'uid=', '-o', 'lstart='], {
|
|
9
|
+
const { stdout } = await exec('/bin/ps', ['-p', String(pid), '-o', 'uid=', '-o', 'lstart='], {
|
|
10
|
+
timeout: 3000,
|
|
11
|
+
maxBuffer: 4096,
|
|
12
|
+
env: { PATH: '/usr/bin:/bin', LC_ALL: 'C' },
|
|
13
|
+
});
|
|
9
14
|
const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(stdout);
|
|
10
15
|
if (!match)
|
|
11
|
-
throw
|
|
16
|
+
throw new SpongeError('unavailable');
|
|
12
17
|
return { pid, uid: Number(match[1]), started: match[2] };
|
|
13
18
|
}
|
|
14
19
|
catch (error) {
|
|
@@ -21,7 +26,10 @@ export async function processIdentity(pid) {
|
|
|
21
26
|
return null;
|
|
22
27
|
}
|
|
23
28
|
}
|
|
24
|
-
throw
|
|
29
|
+
throw new SpongeError('unavailable');
|
|
25
30
|
}
|
|
26
31
|
}
|
|
27
|
-
export async function identityAlive(identity) {
|
|
32
|
+
export async function identityAlive(identity) {
|
|
33
|
+
const current = await processIdentity(identity.pid);
|
|
34
|
+
return current !== null && current.uid === identity.uid && current.started === identity.started;
|
|
35
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export declare const identifier: (value: string) => string;
|
|
2
|
-
export declare function syncDir(path: string): Promise<void>;
|
|
3
2
|
export declare function directory(path: string): Promise<string>;
|
|
3
|
+
/** Atomic replace via a private temp; a failed write or rename removes its temp (a crash leaves it for owner recovery to sweep). */
|
|
4
4
|
export declare function writeJournal(path: string, value: unknown): Promise<void>;
|
|
5
5
|
export declare function readJournal(path: string): Promise<any>;
|
|
@@ -1,30 +1,60 @@
|
|
|
1
1
|
import { constants } from 'node:fs';
|
|
2
|
-
import { open, mkdir, rename, lstat, realpath } from 'node:fs/promises';
|
|
2
|
+
import { open, mkdir, rename, lstat, realpath, rm } from 'node:fs/promises';
|
|
3
3
|
import { join, dirname } from 'node:path';
|
|
4
4
|
import { randomUUID } from 'node:crypto';
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
export
|
|
8
|
-
|
|
5
|
+
import { syncDirectoryAsync } from '../util/fsync.js';
|
|
6
|
+
import { SpongeError } from '../errors.js';
|
|
7
|
+
export const identifier = (value) => {
|
|
8
|
+
if (!/^[a-zA-Z0-9_-]{1,128}$/.test(value))
|
|
9
|
+
throw new SpongeError('invalid-input');
|
|
10
|
+
return value;
|
|
11
|
+
};
|
|
12
|
+
export async function directory(path) {
|
|
13
|
+
await mkdir(path, { recursive: true, mode: 0o700 });
|
|
14
|
+
const stat = await lstat(path);
|
|
15
|
+
if (!stat.isDirectory() ||
|
|
16
|
+
stat.isSymbolicLink() ||
|
|
17
|
+
(await realpath(path)) !== path ||
|
|
18
|
+
stat.uid !== process.getuid?.() ||
|
|
19
|
+
(stat.mode & 0o077) !== 0)
|
|
20
|
+
throw new SpongeError('integrity');
|
|
21
|
+
return path;
|
|
9
22
|
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
23
|
+
/** Atomic replace via a private temp; a failed write or rename removes its temp (a crash leaves it for owner recovery to sweep). */
|
|
24
|
+
export async function writeJournal(path, value) {
|
|
25
|
+
const temp = join(dirname(path), '.' + randomUUID() + '.tmp');
|
|
26
|
+
try {
|
|
27
|
+
const file = await open(temp, 'wx', 0o600);
|
|
28
|
+
try {
|
|
29
|
+
await file.writeFile(JSON.stringify(value));
|
|
30
|
+
await file.sync();
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
await file.close();
|
|
34
|
+
}
|
|
35
|
+
await rename(temp, path);
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
await rm(temp, { force: true });
|
|
39
|
+
throw error;
|
|
40
|
+
}
|
|
41
|
+
await syncDirectoryAsync(dirname(path));
|
|
18
42
|
}
|
|
19
|
-
|
|
20
|
-
await
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
43
|
+
export async function readJournal(path) {
|
|
44
|
+
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
45
|
+
try {
|
|
46
|
+
const s = await file.stat();
|
|
47
|
+
if (!s.isFile() || s.uid !== process.getuid?.() || (s.mode & 0o077) !== 0 || s.size > 65536)
|
|
48
|
+
throw new SpongeError('integrity');
|
|
49
|
+
const text = await file.readFile('utf8');
|
|
50
|
+
try {
|
|
51
|
+
return JSON.parse(text);
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
throw new SpongeError('integrity', { cause: error });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
await file.close();
|
|
59
|
+
}
|
|
27
60
|
}
|
|
28
|
-
finally {
|
|
29
|
-
await file.close();
|
|
30
|
-
} }
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { type ProcessIdentity } from './identity.js';
|
|
2
|
+
/** An exclusive lock file that names its holder. The lock is a hard link of `.<name>.<pid>.<incarnation>.owner`, written
|
|
3
|
+
* before the link exists, so a lock is never visible without its identity. Takeover needs exact proof that the recorded
|
|
4
|
+
* holder is dead (no process with that pid, or a different birth identity); a live, unreadable, legacy or foreign holder
|
|
5
|
+
* stays busy. No age, timeout or pid disappearance alone ever steals it. */
|
|
6
|
+
export interface IdentityLockRecord {
|
|
7
|
+
schemaVersion: 1;
|
|
8
|
+
kind: 'identity-lock';
|
|
9
|
+
incarnation: string;
|
|
10
|
+
identity: ProcessIdentity;
|
|
11
|
+
}
|
|
12
|
+
export interface IdentityLock {
|
|
13
|
+
path: string;
|
|
14
|
+
release: () => Promise<void>;
|
|
15
|
+
}
|
|
16
|
+
/** The recorded holder of `path`, when it is a well-formed identity lock; null for legacy, foreign or unreadable locks. */
|
|
17
|
+
export declare function lockHolder(path: string): Promise<IdentityLockRecord | null>;
|
|
18
|
+
export declare function acquireIdentityLock(path: string): Promise<IdentityLock>;
|