@hraness/sponge 0.2.6 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -9
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +38 -16
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +9 -46
- package/dist/src/digest/editorial.js +57 -49
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +49 -28
- package/dist/src/digest/pipeline.js +369 -102
- package/dist/src/digest/schema.d.ts +11 -25
- package/dist/src/digest/schema.js +86 -18
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +3 -6
- package/dist/src/importers/jungle-digest.js +196 -69
- package/dist/src/importers/jungle-store.d.ts +19 -6
- package/dist/src/importers/jungle-store.js +124 -28
- package/dist/src/importers/jungle.d.ts +14 -14
- package/dist/src/importers/jungle.js +142 -31
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +106 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -79
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { open, link, lstat, rename, unlink, readdir, rm } from 'node:fs/promises';
|
|
3
|
+
import { basename, dirname, join } from 'node:path';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { readJournal } from './journal.js';
|
|
6
|
+
import { syncDirectoryAsync } from '../util/fsync.js';
|
|
7
|
+
import { processIdentity, identityAlive } from './identity.js';
|
|
8
|
+
import { SpongeError } from '../errors.js';
|
|
9
|
+
const code = (error) => error.code;
|
|
10
|
+
const busy = () => new SpongeError('owner-busy');
|
|
11
|
+
const ownerName = (lock, pid, incarnation) => `.${basename(lock)}.${pid}.${incarnation}.owner`;
|
|
12
|
+
const ownerPattern = (lock) => new RegExp(`^\\.${basename(lock).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.(\\d{1,10})\\.([0-9a-f-]{36})\\.owner$`);
|
|
13
|
+
function record(value) {
|
|
14
|
+
return value?.schemaVersion === 1 &&
|
|
15
|
+
value.kind === 'identity-lock' &&
|
|
16
|
+
typeof value.incarnation === 'string' &&
|
|
17
|
+
/^[0-9a-f-]{36}$/.test(value.incarnation) &&
|
|
18
|
+
Number.isSafeInteger(value.identity?.pid) &&
|
|
19
|
+
value.identity.pid > 0 &&
|
|
20
|
+
value.identity.uid === process.getuid?.() &&
|
|
21
|
+
typeof value.identity.started === 'string' &&
|
|
22
|
+
value.identity.started.trim()
|
|
23
|
+
? value
|
|
24
|
+
: null;
|
|
25
|
+
}
|
|
26
|
+
/** Exact death proof, or false: an unavailable process table means unknown, never dead. */
|
|
27
|
+
async function provablyDead(identity) {
|
|
28
|
+
try {
|
|
29
|
+
return !(await identityAlive(identity));
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** The recorded holder of `path`, when it is a well-formed identity lock; null for legacy, foreign or unreadable locks. */
|
|
36
|
+
export async function lockHolder(path) {
|
|
37
|
+
try {
|
|
38
|
+
return record(await readJournal(path));
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/** Remove one stale lock, compare-and-swap style: rename it aside and keep it removed only if it is the exact inode
|
|
45
|
+
* judged dead. Anything else is relinked (if the slot is still free) and reported busy. */
|
|
46
|
+
async function takeOver(path) {
|
|
47
|
+
const dir = dirname(path);
|
|
48
|
+
let before;
|
|
49
|
+
try {
|
|
50
|
+
before = await lstat(path);
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
if (code(error) === 'ENOENT')
|
|
54
|
+
return true;
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
if (!before.isFile() || before.isSymbolicLink())
|
|
58
|
+
return false;
|
|
59
|
+
const holder = await lockHolder(path);
|
|
60
|
+
if (!holder)
|
|
61
|
+
return false;
|
|
62
|
+
try {
|
|
63
|
+
const again = await lstat(path);
|
|
64
|
+
if (again.ino !== before.ino || again.dev !== before.dev)
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
if (code(error) === 'ENOENT')
|
|
69
|
+
return true;
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
if (!(await provablyDead(holder.identity)))
|
|
73
|
+
return false;
|
|
74
|
+
const aside = join(dir, `.${basename(path)}.stale-${randomUUID()}`);
|
|
75
|
+
try {
|
|
76
|
+
await rename(path, aside);
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
if (code(error) === 'ENOENT')
|
|
80
|
+
return true;
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
const moved = await lstat(aside);
|
|
84
|
+
if (moved.ino !== before.ino || moved.dev !== before.dev) {
|
|
85
|
+
try {
|
|
86
|
+
await link(aside, path);
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
/* the slot was taken again; its holder keeps it */
|
|
90
|
+
}
|
|
91
|
+
await unlink(aside);
|
|
92
|
+
await syncDirectoryAsync(dir);
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
await unlink(aside);
|
|
96
|
+
const owner = join(dir, ownerName(path, holder.identity.pid, holder.incarnation));
|
|
97
|
+
try {
|
|
98
|
+
const s = await lstat(owner);
|
|
99
|
+
if (s.isFile() && s.ino === before.ino && s.dev === before.dev)
|
|
100
|
+
await unlink(owner);
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
if (code(error) !== 'ENOENT')
|
|
104
|
+
throw error;
|
|
105
|
+
}
|
|
106
|
+
await syncDirectoryAsync(dir);
|
|
107
|
+
return true;
|
|
108
|
+
}
|
|
109
|
+
/** Owner files that never became (or no longer are) a lock, written by a process that is provably dead. */
|
|
110
|
+
async function sweepOwners(path, keep) {
|
|
111
|
+
const dir = dirname(path), pattern = ownerPattern(path);
|
|
112
|
+
let removed = false;
|
|
113
|
+
for (const name of await readdir(dir)) {
|
|
114
|
+
const match = pattern.exec(name);
|
|
115
|
+
if (!match || name === keep)
|
|
116
|
+
continue;
|
|
117
|
+
const file = join(dir, name);
|
|
118
|
+
try {
|
|
119
|
+
const s = await lstat(file);
|
|
120
|
+
if (!s.isFile() || s.nlink !== 1 || s.uid !== process.getuid?.())
|
|
121
|
+
continue;
|
|
122
|
+
const holder = await lockHolder(file);
|
|
123
|
+
// A torn write has no readable identity: only a pid with no process at all proves its writer is gone.
|
|
124
|
+
if (holder
|
|
125
|
+
? await provablyDead(holder.identity)
|
|
126
|
+
: await processIdentity(Number(match[1])).then((x) => x === null, () => false)) {
|
|
127
|
+
const again = await lstat(file);
|
|
128
|
+
if (again.nlink === 1 && again.ino === s.ino) {
|
|
129
|
+
await unlink(file);
|
|
130
|
+
removed = true;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
if (code(error) !== 'ENOENT')
|
|
136
|
+
throw error;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (removed)
|
|
140
|
+
await syncDirectoryAsync(dir);
|
|
141
|
+
}
|
|
142
|
+
export async function acquireIdentityLock(path) {
|
|
143
|
+
const dir = dirname(path), identity = await processIdentity(process.pid);
|
|
144
|
+
if (!identity)
|
|
145
|
+
throw new SpongeError('unavailable');
|
|
146
|
+
const incarnation = randomUUID(), file = join(dir, ownerName(path, identity.pid, incarnation));
|
|
147
|
+
const handle = await open(file, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
148
|
+
try {
|
|
149
|
+
try {
|
|
150
|
+
await handle.writeFile(JSON.stringify({ schemaVersion: 1, kind: 'identity-lock', incarnation, identity }));
|
|
151
|
+
await handle.sync();
|
|
152
|
+
}
|
|
153
|
+
finally {
|
|
154
|
+
await handle.close();
|
|
155
|
+
}
|
|
156
|
+
for (let attempt = 0;; attempt++) {
|
|
157
|
+
try {
|
|
158
|
+
await link(file, path);
|
|
159
|
+
await syncDirectoryAsync(dir);
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
if (code(error) !== 'EEXIST')
|
|
164
|
+
throw error;
|
|
165
|
+
}
|
|
166
|
+
if (attempt > 1 || !(await takeOver(path)))
|
|
167
|
+
throw busy();
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
await rm(file, { force: true });
|
|
172
|
+
await syncDirectoryAsync(dir);
|
|
173
|
+
throw error;
|
|
174
|
+
}
|
|
175
|
+
await sweepOwners(path, basename(file));
|
|
176
|
+
return {
|
|
177
|
+
path,
|
|
178
|
+
release: async () => {
|
|
179
|
+
let held = false;
|
|
180
|
+
try {
|
|
181
|
+
const [a, b] = await Promise.all([lstat(file), lstat(path)]);
|
|
182
|
+
held = a.ino === b.ino && a.dev === b.dev;
|
|
183
|
+
}
|
|
184
|
+
catch (error) {
|
|
185
|
+
if (code(error) !== 'ENOENT')
|
|
186
|
+
throw error;
|
|
187
|
+
}
|
|
188
|
+
if (held)
|
|
189
|
+
await unlink(path);
|
|
190
|
+
await unlink(file);
|
|
191
|
+
await syncDirectoryAsync(dir);
|
|
192
|
+
if (!held)
|
|
193
|
+
throw new SpongeError('integrity');
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { type LibraryStore } from '../store/index.js';
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
type RuntimeKind = 'capture' | 'model' | 'worker';
|
|
3
|
+
interface RuntimeEntry {
|
|
4
4
|
kind: RuntimeKind;
|
|
5
5
|
name: string;
|
|
6
6
|
bytes: number;
|
|
7
7
|
}
|
|
8
|
-
|
|
8
|
+
interface RuntimeKept {
|
|
9
9
|
kind: RuntimeKind;
|
|
10
10
|
name: string;
|
|
11
11
|
reason: 'referenced' | 'in-use' | 'symlink' | 'foreign' | 'catalog-unavailable' | 'current-missing';
|
|
@@ -28,3 +28,4 @@ export interface RuntimeGcOptions {
|
|
|
28
28
|
* Applying holds worker custody, so no worker, capture, import verification, index run or query
|
|
29
29
|
* can start or be running; each model directory is additionally taken under its setup lock. */
|
|
30
30
|
export declare function collectRuntimes(store: LibraryStore, options: RuntimeGcOptions): Promise<RuntimeGcReport>;
|
|
31
|
+
export {};
|
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
import { constants } from 'node:fs';
|
|
2
2
|
import { lstat, readdir, realpath, rename, rm, link, unlink, open } from 'node:fs/promises';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
5
|
+
import { sha256 } from '../util/hash.js';
|
|
6
6
|
import { modelSetupPlan } from '../models/artifacts.js';
|
|
7
7
|
import { acquireWorkerCustody } from './custody.js';
|
|
8
|
-
import { readJournal, writeJournal
|
|
8
|
+
import { readJournal, writeJournal } from './journal.js';
|
|
9
|
+
import { syncDirectoryAsync } from '../util/fsync.js';
|
|
9
10
|
import { processIdentity, identityAlive } from './identity.js';
|
|
11
|
+
import { SpongeError } from '../errors.js';
|
|
10
12
|
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
11
|
-
const patterns = {
|
|
13
|
+
const patterns = {
|
|
14
|
+
capture: /^capture-[a-z0-9][a-z0-9.-]{0,99}$/,
|
|
15
|
+
model: /^[a-f0-9]{64}$/,
|
|
16
|
+
worker: uuid,
|
|
17
|
+
};
|
|
12
18
|
const helperNames = ['sponge-speech', 'sponge-embedding'];
|
|
13
19
|
const trash = /^\.gc-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
14
20
|
// A worker writes owner.json just after creating its directory; a young unreadable directory may be mid-acquisition.
|
|
15
21
|
const youngMs = 10 * 60 * 1000;
|
|
16
|
-
const integrity = () => new
|
|
22
|
+
const integrity = () => new SpongeError('integrity');
|
|
17
23
|
const missing = (error) => error.code === 'ENOENT';
|
|
18
24
|
/** Disk blocks removing one tree frees, never following links: each inode once, and a file only when all of its
|
|
19
25
|
* hard links are inside the tree (a model artifact shared with a kept package frees nothing). */
|
|
@@ -59,19 +65,25 @@ async function parent(path) {
|
|
|
59
65
|
return null;
|
|
60
66
|
throw error;
|
|
61
67
|
}
|
|
62
|
-
if (!stat.isDirectory() ||
|
|
68
|
+
if (!stat.isDirectory() ||
|
|
69
|
+
stat.isSymbolicLink() ||
|
|
70
|
+
stat.uid !== process.getuid?.() ||
|
|
71
|
+
(stat.mode & 0o077) !== 0 ||
|
|
72
|
+
(await realpath(path)) !== path)
|
|
63
73
|
throw integrity();
|
|
64
74
|
return path;
|
|
65
75
|
}
|
|
66
|
-
async function exists(path) {
|
|
67
|
-
|
|
68
|
-
|
|
76
|
+
async function exists(path) {
|
|
77
|
+
try {
|
|
78
|
+
await lstat(path);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (missing(error))
|
|
83
|
+
return false;
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
69
86
|
}
|
|
70
|
-
catch (error) {
|
|
71
|
-
if (missing(error))
|
|
72
|
-
return false;
|
|
73
|
-
throw error;
|
|
74
|
-
} }
|
|
75
87
|
async function fileSha256(path, maxBytes) {
|
|
76
88
|
let file;
|
|
77
89
|
try {
|
|
@@ -84,7 +96,7 @@ async function fileSha256(path, maxBytes) {
|
|
|
84
96
|
const stat = await file.stat();
|
|
85
97
|
if (!stat.isFile() || stat.size > maxBytes)
|
|
86
98
|
return null;
|
|
87
|
-
return
|
|
99
|
+
return sha256(await file.readFile());
|
|
88
100
|
}
|
|
89
101
|
finally {
|
|
90
102
|
await file.close();
|
|
@@ -123,9 +135,9 @@ async function remove(candidate) {
|
|
|
123
135
|
await rename(moved, path);
|
|
124
136
|
return false;
|
|
125
137
|
}
|
|
126
|
-
await
|
|
138
|
+
await syncDirectoryAsync(candidate.dir);
|
|
127
139
|
await rm(moved, { recursive: true });
|
|
128
|
-
await
|
|
140
|
+
await syncDirectoryAsync(candidate.dir);
|
|
129
141
|
return true;
|
|
130
142
|
}
|
|
131
143
|
async function workerInUse(dir, name) {
|
|
@@ -151,26 +163,30 @@ async function workerInUse(dir, name) {
|
|
|
151
163
|
* can start or be running; each model directory is additionally taken under its setup lock. */
|
|
152
164
|
export async function collectRuntimes(store, options) {
|
|
153
165
|
const dryRun = options.dryRun === true, base = await realpath(store.root), runtime = await parent(join(base, 'runtime'));
|
|
154
|
-
if (store.ownerSnapshot())
|
|
155
|
-
throw new
|
|
156
|
-
if (store.
|
|
157
|
-
throw
|
|
166
|
+
if (store.queue.ownerSnapshot())
|
|
167
|
+
throw new SpongeError('owner-busy');
|
|
168
|
+
if (store.queue.pendingCleanup())
|
|
169
|
+
throw new SpongeError('cleanup-required');
|
|
158
170
|
const lockPath = join(base, 'runtime', 'worker.lock');
|
|
159
171
|
// Checked first so a refused collection leaves no custody journal; acquisition stays the authority.
|
|
160
172
|
if (await exists(lockPath))
|
|
161
|
-
throw new
|
|
173
|
+
throw new SpongeError('owner-busy');
|
|
162
174
|
const custody = dryRun || !runtime ? undefined : await acquireWorkerCustody(base);
|
|
163
175
|
const removed = [], kept = [];
|
|
164
176
|
try {
|
|
165
177
|
if (!runtime)
|
|
166
178
|
return { dryRun, removed, kept, bytes: 0 };
|
|
167
179
|
// Recheck under custody: an owner that appeared before our lock still blocks.
|
|
168
|
-
if (store.ownerSnapshot())
|
|
169
|
-
throw new
|
|
170
|
-
if (store.
|
|
171
|
-
throw
|
|
172
|
-
const dirs = {
|
|
173
|
-
|
|
180
|
+
if (store.queue.ownerSnapshot())
|
|
181
|
+
throw new SpongeError('owner-busy');
|
|
182
|
+
if (store.queue.pendingCleanup())
|
|
183
|
+
throw new SpongeError('cleanup-required');
|
|
184
|
+
const dirs = {
|
|
185
|
+
capture: await parent(join(runtime, 'capture')),
|
|
186
|
+
model: await parent(join(runtime, 'models')),
|
|
187
|
+
worker: await parent(join(runtime, 'workers')),
|
|
188
|
+
};
|
|
189
|
+
const references = store.blobs.runtimeReferences(), models = options.models;
|
|
174
190
|
const modelKeep = new Set(models?.fingerprints ?? []);
|
|
175
191
|
for (const artifact of references.modelArtifacts) {
|
|
176
192
|
if (patterns.model.test(artifact.id))
|
|
@@ -178,22 +194,24 @@ export async function collectRuntimes(store, options) {
|
|
|
178
194
|
try {
|
|
179
195
|
modelKeep.add(modelSetupPlan(JSON.parse(artifact.manifest)).fingerprint);
|
|
180
196
|
}
|
|
181
|
-
catch {
|
|
197
|
+
catch {
|
|
198
|
+
/* An unparsable row still pins its id. */
|
|
199
|
+
}
|
|
182
200
|
}
|
|
183
201
|
// A generation's helper is satisfied only by an installed kept package, never by an uninstalled catalog entry.
|
|
184
202
|
const helpers = new Set(references.embeddingHelpers), installed = new Set();
|
|
185
203
|
for (const fp of modelKeep) {
|
|
186
|
-
if (!dirs.model || !await exists(join(dirs.model, fp)))
|
|
204
|
+
if (!dirs.model || !(await exists(join(dirs.model, fp))))
|
|
187
205
|
continue;
|
|
188
206
|
for (const helper of helperNames) {
|
|
189
207
|
if (await exists(join(dirs.model, fp, helper)))
|
|
190
208
|
installed.add(helper);
|
|
191
209
|
}
|
|
192
|
-
helpers.delete(await fileSha256(join(dirs.model, fp, 'sponge-embedding'), 32 * 1024 ** 2) ?? '');
|
|
210
|
+
helpers.delete((await fileSha256(join(dirs.model, fp, 'sponge-embedding'), 32 * 1024 ** 2)) ?? '');
|
|
193
211
|
}
|
|
194
212
|
const take = async (candidate) => {
|
|
195
213
|
const bytes = await diskBytes(join(candidate.dir, candidate.name));
|
|
196
|
-
if (dryRun || await remove(candidate))
|
|
214
|
+
if (dryRun || (await remove(candidate)))
|
|
197
215
|
removed.push({ kind: candidate.kind, name: candidate.name, bytes });
|
|
198
216
|
else
|
|
199
217
|
kept.push({ kind: candidate.kind, name: candidate.name, reason: 'symlink' });
|
|
@@ -219,21 +237,22 @@ export async function collectRuntimes(store, options) {
|
|
|
219
237
|
continue;
|
|
220
238
|
}
|
|
221
239
|
const helper = join(dir, name, 'sponge-embedding');
|
|
222
|
-
if ((references.unmappedSpaces && await exists(helper)) ||
|
|
240
|
+
if ((references.unmappedSpaces && (await exists(helper))) ||
|
|
241
|
+
(helpers.size && helpers.has((await fileSha256(helper, 32 * 1024 ** 2)) ?? ''))) {
|
|
223
242
|
kept.push({ kind: 'model', name, reason: 'referenced' });
|
|
224
243
|
continue;
|
|
225
244
|
}
|
|
226
245
|
// Until this Sponge's package of a kind is installed, older packages of that kind are its only local copy.
|
|
227
246
|
let missingKind = false;
|
|
228
247
|
for (const helper of helperNames)
|
|
229
|
-
if (!installed.has(helper) && await exists(join(dir, name, helper)))
|
|
248
|
+
if (!installed.has(helper) && (await exists(join(dir, name, helper))))
|
|
230
249
|
missingKind = true;
|
|
231
250
|
if (missingKind) {
|
|
232
251
|
kept.push({ kind: 'model', name, reason: 'current-missing' });
|
|
233
252
|
continue;
|
|
234
253
|
}
|
|
235
254
|
const lock = join(dir, `.${name}.lock`);
|
|
236
|
-
if (await exists(lock) || await exists(join(dir, `.${name}.recovery-lock`))) {
|
|
255
|
+
if ((await exists(lock)) || (await exists(join(dir, `.${name}.recovery-lock`)))) {
|
|
237
256
|
kept.push({ kind: 'model', name, reason: 'in-use' });
|
|
238
257
|
continue;
|
|
239
258
|
}
|
|
@@ -246,7 +265,13 @@ export async function collectRuntimes(store, options) {
|
|
|
246
265
|
if (!identity)
|
|
247
266
|
throw integrity();
|
|
248
267
|
const incarnation = randomUUID(), ownerFile = join(dir, `.model-owner-${incarnation}.json`);
|
|
249
|
-
await writeJournal(ownerFile, {
|
|
268
|
+
await writeJournal(ownerFile, {
|
|
269
|
+
schemaVersion: 1,
|
|
270
|
+
incarnation,
|
|
271
|
+
fingerprint: name,
|
|
272
|
+
stage: '.model-install-' + incarnation,
|
|
273
|
+
identity,
|
|
274
|
+
});
|
|
250
275
|
try {
|
|
251
276
|
try {
|
|
252
277
|
await link(ownerFile, lock);
|
|
@@ -259,7 +284,7 @@ export async function collectRuntimes(store, options) {
|
|
|
259
284
|
throw error;
|
|
260
285
|
}
|
|
261
286
|
try {
|
|
262
|
-
await
|
|
287
|
+
await syncDirectoryAsync(dir);
|
|
263
288
|
await take(candidate);
|
|
264
289
|
}
|
|
265
290
|
finally {
|
|
@@ -268,14 +293,14 @@ export async function collectRuntimes(store, options) {
|
|
|
268
293
|
}
|
|
269
294
|
finally {
|
|
270
295
|
await unlink(ownerFile);
|
|
271
|
-
await
|
|
296
|
+
await syncDirectoryAsync(dir);
|
|
272
297
|
}
|
|
273
298
|
}
|
|
274
299
|
for (const candidate of await children('worker', dirs.worker, kept)) {
|
|
275
300
|
const { name, dir } = candidate;
|
|
276
301
|
if (name === custody?.incarnation)
|
|
277
302
|
continue;
|
|
278
|
-
if (!trash.test(name) && await workerInUse(dir, name)) {
|
|
303
|
+
if (!trash.test(name) && (await workerInUse(dir, name))) {
|
|
279
304
|
kept.push({ kind: 'worker', name, reason: 'in-use' });
|
|
280
305
|
continue;
|
|
281
306
|
}
|
|
@@ -288,9 +313,11 @@ export async function collectRuntimes(store, options) {
|
|
|
288
313
|
await custody.release();
|
|
289
314
|
try {
|
|
290
315
|
await rm(custody.directory, { recursive: true });
|
|
291
|
-
await
|
|
316
|
+
await syncDirectoryAsync(join(base, 'runtime', 'workers'));
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
/* The next collection removes this settled journal. */
|
|
292
320
|
}
|
|
293
|
-
catch { /* The next collection removes this settled journal. */ }
|
|
294
321
|
}
|
|
295
322
|
}
|
|
296
323
|
}
|
|
@@ -13,9 +13,14 @@ const abort = () => controller.abort();
|
|
|
13
13
|
process.on('disconnect', abort);
|
|
14
14
|
process.on('SIGTERM', abort);
|
|
15
15
|
process.on('SIGINT', abort);
|
|
16
|
-
await writeJournal(join(root, ticketId + '.supervisor.json'), {
|
|
17
|
-
|
|
18
|
-
process.
|
|
16
|
+
await writeJournal(join(root, ticketId + '.supervisor.json'), {
|
|
17
|
+
...pending,
|
|
18
|
+
identity: await processIdentity(process.pid),
|
|
19
|
+
});
|
|
20
|
+
const deadline = setTimeout(() => {
|
|
21
|
+
if (!started)
|
|
22
|
+
process.exit(2);
|
|
23
|
+
}, 10000);
|
|
19
24
|
process.once('message', async (message) => {
|
|
20
25
|
if (message?.type !== 'run' || message.ticket !== pending.ticket || message.attemptId !== pending.attemptId)
|
|
21
26
|
process.exit(2);
|
|
@@ -23,11 +28,19 @@ process.once('message', async (message) => {
|
|
|
23
28
|
clearTimeout(deadline);
|
|
24
29
|
const result = await runOwned(message.executable, message.args, { ...message.options, signal: controller.signal });
|
|
25
30
|
if (result.cleanup === 'settled')
|
|
26
|
-
await writeJournal(join(root, ticketId + '.settled.json'), {
|
|
31
|
+
await writeJournal(join(root, ticketId + '.settled.json'), {
|
|
32
|
+
schemaVersion: 1,
|
|
33
|
+
ticket: pending.ticket,
|
|
34
|
+
attemptId: pending.attemptId,
|
|
35
|
+
ownerIncarnation: pending.ownerIncarnation,
|
|
36
|
+
cleanup: 'settled',
|
|
37
|
+
});
|
|
27
38
|
if (process.connected)
|
|
28
39
|
process.send?.({ type: 'result', result }, undefined, undefined, () => process.disconnect());
|
|
29
40
|
});
|
|
30
|
-
process.on('message', (message) => {
|
|
31
|
-
|
|
41
|
+
process.on('message', (message) => {
|
|
42
|
+
if (message?.type === 'cancel')
|
|
43
|
+
abort();
|
|
44
|
+
});
|
|
32
45
|
if (process.connected)
|
|
33
46
|
process.send({ type: 'ready', ticket: pending.ticket });
|
|
@@ -12,19 +12,41 @@ export function supervisedRunner(ownerDirectory, ownerIncarnation, attemptId) {
|
|
|
12
12
|
const root = await directory(join(ownerDirectory, 'attempts', attemptId)), ticket = randomUUID();
|
|
13
13
|
await writeJournal(join(root, ticket + '.pending.json'), { schemaVersion: 1, ticket, attemptId, ownerIncarnation });
|
|
14
14
|
const source = import.meta.url.endsWith('.ts'), entry = fileURLToPath(new URL(source ? './supervisor-child.ts' : './supervisor-child.js', import.meta.url));
|
|
15
|
-
return new Promise(resolve => {
|
|
16
|
-
const child = fork(entry, [root, ticket], {
|
|
15
|
+
return new Promise((resolve) => {
|
|
16
|
+
const child = fork(entry, [root, ticket], {
|
|
17
|
+
detached: true,
|
|
18
|
+
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
19
|
+
execArgv: source ? ['--import', import.meta.resolve('tsx')] : [],
|
|
20
|
+
env: { PATH: '/usr/bin:/bin', LANG: 'en_US.UTF-8' },
|
|
21
|
+
});
|
|
17
22
|
let result, finished = false, dispatched = false;
|
|
18
|
-
const cancel = () => {
|
|
19
|
-
|
|
23
|
+
const cancel = () => {
|
|
24
|
+
if (dispatched && child.connected)
|
|
25
|
+
child.send({ type: 'cancel' }, () => { });
|
|
26
|
+
};
|
|
20
27
|
options.signal?.addEventListener('abort', cancel, { once: true });
|
|
21
28
|
child.on('message', (message) => {
|
|
22
29
|
if (message?.type === 'result' && dispatched)
|
|
23
30
|
result = message.result;
|
|
24
31
|
if (message?.type === 'ready' && message.ticket === ticket && !dispatched) {
|
|
25
32
|
dispatched = true;
|
|
26
|
-
child.send({
|
|
27
|
-
|
|
33
|
+
child.send({
|
|
34
|
+
type: 'run',
|
|
35
|
+
ticket,
|
|
36
|
+
attemptId,
|
|
37
|
+
executable,
|
|
38
|
+
args,
|
|
39
|
+
options: {
|
|
40
|
+
cwd: options.cwd,
|
|
41
|
+
env: options.env,
|
|
42
|
+
deadlineMs: options.deadlineMs,
|
|
43
|
+
maxOutputBytes: options.maxOutputBytes,
|
|
44
|
+
graceMs: options.graceMs,
|
|
45
|
+
},
|
|
46
|
+
}, () => {
|
|
47
|
+
if (options.signal?.aborted)
|
|
48
|
+
cancel();
|
|
49
|
+
});
|
|
28
50
|
}
|
|
29
51
|
});
|
|
30
52
|
const finish = async () => {
|
|
@@ -35,10 +57,21 @@ export function supervisedRunner(ownerDirectory, ownerIncarnation, attemptId) {
|
|
|
35
57
|
let settled = false;
|
|
36
58
|
try {
|
|
37
59
|
const receipt = await readJournal(join(root, ticket + '.settled.json'));
|
|
38
|
-
settled =
|
|
60
|
+
settled =
|
|
61
|
+
receipt.ticket === ticket &&
|
|
62
|
+
receipt.attemptId === attemptId &&
|
|
63
|
+
receipt.ownerIncarnation === ownerIncarnation &&
|
|
64
|
+
receipt.cleanup === 'settled';
|
|
39
65
|
}
|
|
40
66
|
catch { }
|
|
41
|
-
resolve(result && settled
|
|
67
|
+
resolve(result && settled
|
|
68
|
+
? result
|
|
69
|
+
: {
|
|
70
|
+
code: null,
|
|
71
|
+
stdout: '',
|
|
72
|
+
stopped: options.signal?.aborted ? 'cancelled' : 'unavailable',
|
|
73
|
+
cleanup: 'pending',
|
|
74
|
+
});
|
|
42
75
|
};
|
|
43
76
|
child.once('error', () => void finish());
|
|
44
77
|
child.once('exit', () => void finish());
|
|
@@ -27,14 +27,14 @@
|
|
|
27
27
|
}
|
|
28
28
|
],
|
|
29
29
|
"build": {
|
|
30
|
-
"asrSourceSha256": "
|
|
30
|
+
"asrSourceSha256": "94b9303afcf9304e695c37411cef065e9cc45a31e0e51cd171819cbfc9ee3955",
|
|
31
31
|
"clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
|
|
32
|
-
"embeddingSourceSha256": "
|
|
32
|
+
"embeddingSourceSha256": "eed7e110bc888af99e0d6e614275cea7a7dbaa56f88d3de9bdd04cc9ff4cfa45",
|
|
33
33
|
"swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
|
|
34
34
|
},
|
|
35
35
|
"minimumMacOS": "26.0",
|
|
36
|
-
"packageVersion": "0.
|
|
36
|
+
"packageVersion": "0.3.1",
|
|
37
37
|
"platform": "darwin",
|
|
38
38
|
"schemaVersion": 1,
|
|
39
|
-
"sourceCommit": "
|
|
39
|
+
"sourceCommit": "b8975aff1dc06e398c3f78811e76f4ff875729a1"
|
|
40
40
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hraness/sponge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "A local library for captured articles and verified offline assets.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -30,6 +30,8 @@
|
|
|
30
30
|
"check": "node --check bin/sponge.js",
|
|
31
31
|
"build": "tsc -p tsconfig.build.json",
|
|
32
32
|
"typecheck": "tsc --noEmit",
|
|
33
|
+
"cycles": "madge --circular --extensions ts src",
|
|
34
|
+
"deadcode": "knip --no-progress",
|
|
33
35
|
"test": "node --import tsx --test --test-isolation=none test/*/*.test.ts",
|
|
34
36
|
"probe": "node --import tsx scripts/probes/cli.ts",
|
|
35
37
|
"prepack": "npm run build",
|
|
@@ -38,11 +40,17 @@
|
|
|
38
40
|
"verify:pdf": "node --import tsx scripts/verify-pdf.ts",
|
|
39
41
|
"verify:import": "node --import tsx scripts/verify-import.ts",
|
|
40
42
|
"verify:installed": "node --import tsx scripts/verify-installed.ts",
|
|
41
|
-
"verify:installed-native": "node --import tsx scripts/verify-installed-native.ts"
|
|
43
|
+
"verify:installed-native": "node --import tsx scripts/verify-installed-native.ts",
|
|
44
|
+
"format": "prettier --write \"{bin,contracts,migrations,scripts,src,test}/**/*.{ts,js,mjs}\"",
|
|
45
|
+
"format:check": "prettier --check \"{bin,contracts,migrations,scripts,src,test}/**/*.{ts,js,mjs}\""
|
|
42
46
|
},
|
|
43
47
|
"packageManager": "npm@11.19.0",
|
|
44
48
|
"devDependencies": {
|
|
49
|
+
"@types/mdast": "4.0.4",
|
|
45
50
|
"@types/node": "24.10.1",
|
|
51
|
+
"knip": "6.39.0",
|
|
52
|
+
"madge": "8.0.0",
|
|
53
|
+
"prettier": "3.9.10",
|
|
46
54
|
"tsx": "4.20.6",
|
|
47
55
|
"typescript": "5.9.3"
|
|
48
56
|
},
|