@hraness/sponge 0.3.5 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -7
- package/dist/migrations/011.d.ts +3 -0
- package/dist/migrations/011.js +9 -0
- package/dist/migrations/index.js +2 -0
- package/dist/src/cli/commands.js +59 -7
- package/dist/src/cli/runtimes.js +6 -2
- package/dist/src/cli/worker.js +5 -1
- package/dist/src/digest/endpoint.d.ts +3 -2
- package/dist/src/digest/endpoint.js +3 -2
- package/dist/src/digest/pipeline.d.ts +29 -4
- package/dist/src/digest/pipeline.js +125 -21
- package/dist/src/digest/runner.d.ts +33 -0
- package/dist/src/digest/runner.js +82 -0
- package/dist/src/digest/server.d.ts +40 -0
- package/dist/src/digest/server.js +166 -0
- package/dist/src/errors.d.ts +2 -0
- package/dist/src/errors.js +4 -0
- package/dist/src/importers/custody.d.ts +2 -5
- package/dist/src/importers/custody.js +3 -51
- package/dist/src/models/artifacts.d.ts +5 -0
- package/dist/src/models/artifacts.js +92 -17
- package/dist/src/models/digest.d.ts +64 -0
- package/dist/src/models/digest.js +122 -0
- package/dist/src/portability/database.js +2 -0
- package/dist/src/search/query.js +7 -54
- package/dist/src/store/digest-jobs.d.ts +58 -0
- package/dist/src/store/digest-jobs.js +168 -0
- package/dist/src/store/library.d.ts +3 -0
- package/dist/src/store/library.js +5 -1
- package/dist/src/store/queue.d.ts +9 -5
- package/dist/src/store/queue.js +14 -9
- package/dist/src/store/revisions.js +4 -0
- package/dist/src/worker/custody.d.ts +5 -0
- package/dist/src/worker/custody.js +7 -4
- package/dist/src/worker/transient.d.ts +17 -0
- package/dist/src/worker/transient.js +60 -0
- package/native/runtime-manifest.json +4 -4
- package/package.json +1 -1
|
@@ -116,6 +116,9 @@ async function sweepTemps(path, depth = 2) {
|
|
|
116
116
|
if (removed)
|
|
117
117
|
await syncDirectoryAsync(path);
|
|
118
118
|
}
|
|
119
|
+
/** Attempts outside the job tables: a search query, an import verification, a digest server. Each writes `KIND.json`
|
|
120
|
+
* under its attempt and `staging/KIND-ID/ownership.json`. */
|
|
121
|
+
const transientKinds = ['query', 'import-verification', 'digest-server'];
|
|
119
122
|
export async function recoverWorker(store, options = {}) {
|
|
120
123
|
const lock = join(store.root, 'runtime', 'worker.lock');
|
|
121
124
|
let record;
|
|
@@ -171,7 +174,7 @@ export async function recoverWorker(store, options = {}) {
|
|
|
171
174
|
identifier(id);
|
|
172
175
|
if (!attemptIds.has(id)) {
|
|
173
176
|
let marker;
|
|
174
|
-
for (const kind of
|
|
177
|
+
for (const kind of transientKinds) {
|
|
175
178
|
try {
|
|
176
179
|
const candidate = await readJournal(join(ownerDirectory, 'attempts', id, kind + '.json'));
|
|
177
180
|
if (marker || candidate.kind !== kind)
|
|
@@ -189,7 +192,7 @@ export async function recoverWorker(store, options = {}) {
|
|
|
189
192
|
}
|
|
190
193
|
else {
|
|
191
194
|
if (marker.schemaVersion !== 1 ||
|
|
192
|
-
!
|
|
195
|
+
!transientKinds.includes(marker.kind) ||
|
|
193
196
|
marker.attemptId !== id ||
|
|
194
197
|
marker.ownerIncarnation !== record.incarnation)
|
|
195
198
|
throw pending();
|
|
@@ -201,7 +204,7 @@ export async function recoverWorker(store, options = {}) {
|
|
|
201
204
|
}
|
|
202
205
|
const staging = join(store.root, 'staging');
|
|
203
206
|
for (const entry of await readdir(staging)) {
|
|
204
|
-
if (
|
|
207
|
+
if (transientKinds.some((kind) => entry.startsWith(kind + '-'))) {
|
|
205
208
|
const path = join(staging, entry);
|
|
206
209
|
await directory(path);
|
|
207
210
|
let marker;
|
|
@@ -212,7 +215,7 @@ export async function recoverWorker(store, options = {}) {
|
|
|
212
215
|
throw pending();
|
|
213
216
|
}
|
|
214
217
|
if (marker.schemaVersion !== 1 ||
|
|
215
|
-
!
|
|
218
|
+
!transientKinds.includes(marker.kind) ||
|
|
216
219
|
typeof marker.requestId !== 'string' ||
|
|
217
220
|
entry !== marker.kind + '-' + marker.requestId ||
|
|
218
221
|
typeof marker.ownerIncarnation !== 'string')
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type TransientKind } from './custody.js';
|
|
2
|
+
import { type ProcessRunner } from './supervisor.js';
|
|
3
|
+
export interface TransientAttempt {
|
|
4
|
+
attemptId: string;
|
|
5
|
+
/** The custody incarnation, for `queue.acquireOwner` when the attempt also claims jobs. */
|
|
6
|
+
incarnation: string;
|
|
7
|
+
ownerDirectory: string;
|
|
8
|
+
scratchRoot: string;
|
|
9
|
+
/** Runs helpers under the worker supervisor, so their process groups are proved collected. */
|
|
10
|
+
runProcess: ProcessRunner;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* One custody-held attempt outside the job tables. It writes `KIND.json` under the attempt and an ownership marker in
|
|
14
|
+
* `staging/KIND-ID` (what `recoverWorker` expects), proves every helper collected, removes the scratch root and
|
|
15
|
+
* releases custody. Unproven cleanup keeps custody and throws `cleanup-required`. Mutates coordination journals only.
|
|
16
|
+
*/
|
|
17
|
+
export declare function withTransientAttempt<T>(libraryRoot: string, kind: TransientKind, fn: (attempt: TransientAttempt) => Promise<T>): Promise<T>;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { rm } from 'node:fs/promises';
|
|
4
|
+
import { acquireWorkerCustody, proveAttemptCleanup } from './custody.js';
|
|
5
|
+
import { directory, writeJournal, readJournal } from './journal.js';
|
|
6
|
+
import { supervisedRunner } from './supervisor.js';
|
|
7
|
+
import { syncDirectoryAsync } from '../util/fsync.js';
|
|
8
|
+
import { SpongeError, codeOf } from '../errors.js';
|
|
9
|
+
/**
|
|
10
|
+
* One custody-held attempt outside the job tables. It writes `KIND.json` under the attempt and an ownership marker in
|
|
11
|
+
* `staging/KIND-ID` (what `recoverWorker` expects), proves every helper collected, removes the scratch root and
|
|
12
|
+
* releases custody. Unproven cleanup keeps custody and throws `cleanup-required`. Mutates coordination journals only.
|
|
13
|
+
*/
|
|
14
|
+
export async function withTransientAttempt(libraryRoot, kind, fn) {
|
|
15
|
+
const custody = await acquireWorkerCustody(libraryRoot), attemptId = randomUUID();
|
|
16
|
+
let scratchRoot, cleanupUnknown = false;
|
|
17
|
+
try {
|
|
18
|
+
const attempts = await directory(join(custody.directory, 'attempts', attemptId));
|
|
19
|
+
await writeJournal(join(attempts, kind + '.json'), {
|
|
20
|
+
schemaVersion: 1,
|
|
21
|
+
kind,
|
|
22
|
+
attemptId,
|
|
23
|
+
ownerIncarnation: custody.incarnation,
|
|
24
|
+
});
|
|
25
|
+
scratchRoot = await directory(join(libraryRoot, 'staging', `${kind}-${attemptId}`));
|
|
26
|
+
await writeJournal(join(scratchRoot, 'ownership.json'), {
|
|
27
|
+
schemaVersion: 1,
|
|
28
|
+
kind,
|
|
29
|
+
requestId: attemptId,
|
|
30
|
+
ownerIncarnation: custody.incarnation,
|
|
31
|
+
});
|
|
32
|
+
return await fn({
|
|
33
|
+
attemptId,
|
|
34
|
+
incarnation: custody.incarnation,
|
|
35
|
+
ownerDirectory: custody.directory,
|
|
36
|
+
scratchRoot,
|
|
37
|
+
runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId),
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
cleanupUnknown = codeOf(error) === 'cleanup-required';
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
const proved = !cleanupUnknown && (await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId));
|
|
46
|
+
if (!proved)
|
|
47
|
+
throw new SpongeError('cleanup-required');
|
|
48
|
+
if (scratchRoot) {
|
|
49
|
+
const marker = await readJournal(join(scratchRoot, 'ownership.json'));
|
|
50
|
+
if (marker.schemaVersion !== 1 ||
|
|
51
|
+
marker.kind !== kind ||
|
|
52
|
+
marker.requestId !== attemptId ||
|
|
53
|
+
marker.ownerIncarnation !== custody.incarnation)
|
|
54
|
+
throw new SpongeError('cleanup-required');
|
|
55
|
+
await rm(scratchRoot, { recursive: true });
|
|
56
|
+
await syncDirectoryAsync(join(libraryRoot, 'staging'));
|
|
57
|
+
}
|
|
58
|
+
await custody.release();
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -27,14 +27,14 @@
|
|
|
27
27
|
}
|
|
28
28
|
],
|
|
29
29
|
"build": {
|
|
30
|
-
"asrSourceSha256": "
|
|
30
|
+
"asrSourceSha256": "6a84b1b2040f17cb510a71548c1453442df15f820ceb0ac8cec3086b4ccefbdc",
|
|
31
31
|
"clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
|
|
32
|
-
"embeddingSourceSha256": "
|
|
32
|
+
"embeddingSourceSha256": "b029dadccaeba029fb9ef302daaefc6a134e8d46440c1e8c227f374d97202ce3",
|
|
33
33
|
"swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
|
|
34
34
|
},
|
|
35
35
|
"minimumMacOS": "26.0",
|
|
36
|
-
"packageVersion": "0.
|
|
36
|
+
"packageVersion": "0.4.1",
|
|
37
37
|
"platform": "darwin",
|
|
38
38
|
"schemaVersion": 1,
|
|
39
|
-
"sourceCommit": "
|
|
39
|
+
"sourceCommit": "82fefa4ca18700e2e7390e1372c7c672ddac11fa"
|
|
40
40
|
}
|