@hraness/sponge 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -9
- package/bin/sponge.js +8 -7
- package/dist/contracts/index.js +24 -6
- package/dist/contracts/validate.js +106 -29
- package/dist/migrations/008.d.ts +3 -0
- package/dist/migrations/008.js +5 -0
- package/dist/migrations/009.d.ts +3 -0
- package/dist/migrations/009.js +8 -0
- package/dist/migrations/010.d.ts +3 -0
- package/dist/migrations/010.js +11 -0
- package/dist/migrations/index.d.ts +15 -0
- package/dist/migrations/index.js +39 -0
- package/dist/src/acquisition/capture.js +87 -28
- package/dist/src/acquisition/cleanup.d.ts +0 -4
- package/dist/src/acquisition/cleanup.js +3 -5
- package/dist/src/acquisition/errors.js +17 -2
- package/dist/src/acquisition/files.d.ts +0 -1
- package/dist/src/acquisition/files.js +14 -11
- package/dist/src/acquisition/index.d.ts +2 -3
- package/dist/src/acquisition/index.js +1 -2
- package/dist/src/acquisition/lock.js +1 -1
- package/dist/src/acquisition/normalize.js +57 -18
- package/dist/src/acquisition/pins.js +22 -4
- package/dist/src/acquisition/process.js +67 -29
- package/dist/src/acquisition/runtime.d.ts +5 -7
- package/dist/src/acquisition/runtime.js +129 -68
- package/dist/src/acquisition/verify-helper.js +10 -3
- package/dist/src/annotations/quote.js +40 -12
- package/dist/src/annotations/store.d.ts +0 -1
- package/dist/src/annotations/store.js +66 -14
- package/dist/src/app/article.d.ts +2 -1
- package/dist/src/app/article.js +66 -70
- package/dist/src/app/bundle-files.d.ts +0 -4
- package/dist/src/app/bundle-files.js +11 -14
- package/dist/src/app/markdown.js +18 -6
- package/dist/src/app/source.d.ts +0 -4
- package/dist/src/app/source.js +18 -14
- package/dist/src/cli/arguments.d.ts +2 -8
- package/dist/src/cli/arguments.js +67 -80
- package/dist/src/cli/commands.d.ts +42 -0
- package/dist/src/cli/commands.js +795 -0
- package/dist/src/cli/config.js +12 -9
- package/dist/src/cli/digest.d.ts +1 -1
- package/dist/src/cli/digest.js +38 -16
- package/dist/src/cli/import.js +25 -4
- package/dist/src/cli/indexing.d.ts +2 -0
- package/dist/src/cli/indexing.js +26 -9
- package/dist/src/cli/main.js +47 -386
- package/dist/src/cli/portability.d.ts +0 -2
- package/dist/src/cli/portability.js +13 -8
- package/dist/src/cli/queue.js +12 -8
- package/dist/src/cli/runtimes.d.ts +1 -3
- package/dist/src/cli/runtimes.js +13 -6
- package/dist/src/cli/terminal.js +1 -1
- package/dist/src/cli/worker.d.ts +7 -0
- package/dist/src/cli/worker.js +54 -15
- package/dist/src/derivations/kinds.d.ts +34 -0
- package/dist/src/derivations/kinds.js +14 -0
- package/dist/src/derivations/types.d.ts +48 -0
- package/dist/src/derivations/types.js +1 -0
- package/dist/src/digest/derivation.d.ts +25 -0
- package/dist/src/digest/derivation.js +124 -0
- package/dist/src/digest/editorial.d.ts +9 -46
- package/dist/src/digest/editorial.js +57 -49
- package/dist/src/digest/endpoint.js +63 -27
- package/dist/src/digest/pipeline.d.ts +49 -28
- package/dist/src/digest/pipeline.js +343 -100
- package/dist/src/digest/schema.d.ts +11 -25
- package/dist/src/digest/schema.js +86 -18
- package/dist/src/errors.d.ts +46 -0
- package/dist/src/errors.js +117 -0
- package/dist/src/identity/query-policy.d.ts +0 -5
- package/dist/src/identity/query-policy.js +61 -9
- package/dist/src/identity/uri.d.ts +6 -5
- package/dist/src/identity/uri.js +37 -18
- package/dist/src/identity/visibility.d.ts +11 -1
- package/dist/src/identity/visibility.js +81 -17
- package/dist/src/importers/apply.js +104 -36
- package/dist/src/importers/custody.js +27 -9
- package/dist/src/importers/inspect.js +125 -38
- package/dist/src/importers/jungle-digest.d.ts +3 -6
- package/dist/src/importers/jungle-digest.js +196 -69
- package/dist/src/importers/jungle-store.d.ts +15 -6
- package/dist/src/importers/jungle-store.js +122 -28
- package/dist/src/importers/jungle.d.ts +14 -14
- package/dist/src/importers/jungle.js +136 -31
- package/dist/src/importers/manifest.js +57 -21
- package/dist/src/importers/verify-helper.js +18 -4
- package/dist/src/indexing/image.js +29 -10
- package/dist/src/indexing/index.d.ts +37 -6
- package/dist/src/indexing/index.js +360 -124
- package/dist/src/indexing/jobs.js +156 -47
- package/dist/src/indexing/prune-run.d.ts +2 -0
- package/dist/src/indexing/prune-run.js +69 -26
- package/dist/src/indexing/prune.d.ts +12 -3
- package/dist/src/indexing/prune.js +85 -35
- package/dist/src/indexing/runner.d.ts +2 -1
- package/dist/src/indexing/runner.js +83 -40
- package/dist/src/indexing/upgrade.d.ts +9 -0
- package/dist/src/indexing/upgrade.js +79 -0
- package/dist/src/inference/asr/bridge.js +6 -2
- package/dist/src/inference/asr/protocol.js +109 -19
- package/dist/src/inference/asr/runtime.js +58 -21
- package/dist/src/inference/asr/windows.js +9 -5
- package/dist/src/inference/embedding/index.d.ts +2 -6
- package/dist/src/inference/embedding/index.js +89 -29
- package/dist/src/ingest/attempt.d.ts +29 -0
- package/dist/src/ingest/attempt.js +55 -0
- package/dist/src/media/capture-helper.js +7 -2
- package/dist/src/media/capture.d.ts +2 -1
- package/dist/src/media/capture.js +124 -43
- package/dist/src/media/handler.d.ts +4 -2
- package/dist/src/media/handler.js +88 -70
- package/dist/src/media/transcribe.js +63 -21
- package/dist/src/metadata/article.js +49 -17
- package/dist/src/models/artifacts.d.ts +10 -8
- package/dist/src/models/artifacts.js +237 -106
- package/dist/src/models/catalog.d.ts +2 -5
- package/dist/src/models/catalog.js +94 -25
- package/dist/src/models/download.js +55 -15
- package/dist/src/models/runtime.d.ts +0 -2
- package/dist/src/models/runtime.js +69 -33
- package/dist/src/pdf/capture-helper.js +3 -1
- package/dist/src/pdf/capture.d.ts +2 -1
- package/dist/src/pdf/capture.js +95 -27
- package/dist/src/pdf/errors.js +12 -1
- package/dist/src/pdf/handler.d.ts +2 -1
- package/dist/src/pdf/handler.js +70 -71
- package/dist/src/pdf/index.d.ts +1 -3
- package/dist/src/pdf/index.js +0 -2
- package/dist/src/pdf/input.js +12 -7
- package/dist/src/pdf/normalize.js +80 -21
- package/dist/src/pdf/source-identity.js +3 -2
- package/dist/src/pdf/source.js +12 -8
- package/dist/src/pdf/verify-helper.js +15 -3
- package/dist/src/portability/backup.d.ts +2 -9
- package/dist/src/portability/backup.js +106 -22
- package/dist/src/portability/database.js +51 -17
- package/dist/src/portability/export.js +35 -9
- package/dist/src/portability/files.d.ts +1 -4
- package/dist/src/portability/files.js +127 -67
- package/dist/src/portability/gc.js +65 -22
- package/dist/src/publish/component.js +1 -1
- package/dist/src/publish/feed.d.ts +2 -2
- package/dist/src/publish/feed.js +75 -29
- package/dist/src/publish/identity.d.ts +5 -1
- package/dist/src/publish/identity.js +31 -11
- package/dist/src/publish/index.d.ts +2 -3
- package/dist/src/publish/index.js +34 -21
- package/dist/src/publish/projection.d.ts +2 -9
- package/dist/src/publish/projection.js +21 -14
- package/dist/src/search/extension.js +3 -1
- package/dist/src/search/hybrid.d.ts +3 -8
- package/dist/src/search/hybrid.js +45 -11
- package/dist/src/search/lexical.js +18 -4
- package/dist/src/search/query.js +56 -25
- package/dist/src/search/vector.js +37 -10
- package/dist/src/service/launch-agent.d.ts +5 -3
- package/dist/src/service/launch-agent.js +272 -83
- package/dist/src/service/lifetime.js +47 -14
- package/dist/src/service/loop.d.ts +3 -3
- package/dist/src/service/loop.js +47 -19
- package/dist/src/store/annotations.d.ts +32 -0
- package/dist/src/store/annotations.js +44 -0
- package/dist/src/store/backups.d.ts +18 -0
- package/dist/src/store/backups.js +110 -0
- package/dist/src/store/blobs.d.ts +24 -0
- package/dist/src/store/blobs.js +139 -0
- package/dist/src/store/changes.d.ts +5 -12
- package/dist/src/store/changes.js +108 -56
- package/dist/src/store/context.d.ts +15 -0
- package/dist/src/store/context.js +19 -0
- package/dist/src/store/derivations.d.ts +42 -0
- package/dist/src/store/derivations.js +104 -0
- package/dist/src/store/files.d.ts +0 -2
- package/dist/src/store/files.js +19 -24
- package/dist/src/store/ids.d.ts +30 -0
- package/dist/src/store/ids.js +34 -0
- package/dist/src/store/imports.d.ts +26 -0
- package/dist/src/store/imports.js +57 -0
- package/dist/src/store/index.d.ts +3 -186
- package/dist/src/store/index.js +2 -809
- package/dist/src/store/lease.d.ts +20 -0
- package/dist/src/store/lease.js +62 -0
- package/dist/src/store/library.d.ts +75 -0
- package/dist/src/store/library.js +174 -0
- package/dist/src/store/open.d.ts +7 -0
- package/dist/src/store/open.js +183 -0
- package/dist/src/store/queue.d.ts +65 -0
- package/dist/src/store/queue.js +327 -0
- package/dist/src/store/revisions.d.ts +12 -0
- package/dist/src/store/revisions.js +189 -0
- package/dist/src/store/search-index.d.ts +63 -0
- package/dist/src/store/search-index.js +190 -0
- package/dist/src/store/sync-ready.d.ts +6 -0
- package/dist/src/store/sync-ready.js +121 -0
- package/dist/src/store/types.d.ts +1 -5
- package/dist/src/store/types.js +1 -8
- package/dist/src/util/canonical.d.ts +2 -0
- package/dist/src/util/canonical.js +17 -0
- package/dist/src/util/exact.d.ts +2 -0
- package/dist/src/util/exact.js +9 -0
- package/dist/src/util/fsync.d.ts +4 -0
- package/dist/src/util/fsync.js +22 -0
- package/dist/src/util/hash.d.ts +4 -0
- package/dist/src/util/hash.js +8 -0
- package/dist/src/worker/basic.d.ts +0 -1
- package/dist/src/worker/basic.js +23 -7
- package/dist/src/worker/custody.d.ts +4 -1
- package/dist/src/worker/custody.js +111 -38
- package/dist/src/worker/engine.d.ts +2 -1
- package/dist/src/worker/engine.js +42 -20
- package/dist/src/worker/identity.js +13 -5
- package/dist/src/worker/journal.d.ts +1 -1
- package/dist/src/worker/journal.js +54 -24
- package/dist/src/worker/lock.d.ts +18 -0
- package/dist/src/worker/lock.js +196 -0
- package/dist/src/worker/runtime-gc.d.ts +4 -3
- package/dist/src/worker/runtime-gc.js +68 -41
- package/dist/src/worker/supervisor-child.js +19 -6
- package/dist/src/worker/supervisor.js +41 -8
- package/native/runtime-manifest.json +4 -4
- package/package.json +10 -2
- package/dist/src/digest/store.d.ts +0 -57
- package/dist/src/digest/store.js +0 -79
- package/dist/src/store/foundation.d.ts +0 -7
- package/dist/src/store/foundation.js +0 -7
|
@@ -4,18 +4,18 @@ import { dirname, join } from 'node:path';
|
|
|
4
4
|
import { createHash, randomUUID } from 'node:crypto';
|
|
5
5
|
import { processIdentity, identityAlive } from '../worker/identity.js';
|
|
6
6
|
import { readJournal, writeJournal } from '../worker/journal.js';
|
|
7
|
+
import { acquireIdentityLock } from '../worker/lock.js';
|
|
8
|
+
import { syncDirectoryAsync } from '../util/fsync.js';
|
|
7
9
|
import { privateRoot } from '../acquisition/files.js';
|
|
8
|
-
|
|
9
|
-
code;
|
|
10
|
-
constructor(code) {
|
|
11
|
-
super(code);
|
|
12
|
-
this.code = code;
|
|
13
|
-
}
|
|
14
|
-
}
|
|
10
|
+
import { SpongeError, codeOf } from '../errors.js';
|
|
15
11
|
const maximumBytes = 2 * 1024 ** 3;
|
|
16
12
|
function validate(manifest) {
|
|
17
|
-
if (!/^[a-z0-9][a-z0-9.-]{0,127}$/.test(manifest.id) ||
|
|
18
|
-
|
|
13
|
+
if (!/^[a-z0-9][a-z0-9.-]{0,127}$/.test(manifest.id) ||
|
|
14
|
+
!manifest.license ||
|
|
15
|
+
manifest.license.length > 256 ||
|
|
16
|
+
!manifest.artifacts.length ||
|
|
17
|
+
manifest.artifacts.length > 16)
|
|
18
|
+
throw new SpongeError('invalid-input');
|
|
19
19
|
const names = new Set();
|
|
20
20
|
let bytes = 0;
|
|
21
21
|
for (const artifact of manifest.artifacts) {
|
|
@@ -24,49 +24,73 @@ function validate(manifest) {
|
|
|
24
24
|
url = new URL(artifact.url);
|
|
25
25
|
}
|
|
26
26
|
catch {
|
|
27
|
-
throw new
|
|
27
|
+
throw new SpongeError('invalid-input');
|
|
28
28
|
}
|
|
29
|
-
if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(artifact.name) ||
|
|
30
|
-
|
|
29
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(artifact.name) ||
|
|
30
|
+
names.has(artifact.name) ||
|
|
31
|
+
url.protocol !== 'https:' ||
|
|
32
|
+
url.username ||
|
|
33
|
+
url.password ||
|
|
34
|
+
!/^[a-f0-9]{64}$/.test(artifact.sha256) ||
|
|
35
|
+
!Number.isSafeInteger(artifact.bytes) ||
|
|
36
|
+
artifact.bytes < 1 ||
|
|
37
|
+
artifact.bytes > maximumBytes)
|
|
38
|
+
throw new SpongeError('invalid-input');
|
|
31
39
|
names.add(artifact.name);
|
|
32
40
|
bytes += artifact.bytes;
|
|
33
41
|
}
|
|
34
42
|
if (bytes > maximumBytes)
|
|
35
|
-
throw new
|
|
43
|
+
throw new SpongeError('limit-exceeded');
|
|
36
44
|
}
|
|
45
|
+
/** Installed-package identity is content only: artifact names, bytes, sha256 and executable bit. The package id,
|
|
46
|
+
* license text and per-release download URLs are metadata, so a release that ships byte-identical artifacts keeps
|
|
47
|
+
* the installed directory. The domain tag keeps these names disjoint from the earlier URL-bound fingerprints. */
|
|
37
48
|
function identity(manifest) {
|
|
38
|
-
return createHash('sha256')
|
|
49
|
+
return createHash('sha256')
|
|
50
|
+
.update(JSON.stringify({
|
|
51
|
+
identity: 'sponge-model-content-v1',
|
|
52
|
+
artifacts: manifest.artifacts
|
|
53
|
+
.map((a) => ({ name: a.name, sha256: a.sha256, bytes: a.bytes, executable: !!a.executable }))
|
|
54
|
+
.sort((a, b) => a.name.localeCompare(b.name)),
|
|
55
|
+
}))
|
|
56
|
+
.digest('hex');
|
|
39
57
|
}
|
|
40
58
|
export function modelSetupPlan(manifest) {
|
|
41
59
|
validate(manifest);
|
|
42
60
|
const downloadBytes = manifest.artifacts.reduce((n, a) => n + a.bytes, 0);
|
|
43
|
-
return {
|
|
61
|
+
return {
|
|
62
|
+
id: manifest.id,
|
|
63
|
+
license: manifest.license,
|
|
64
|
+
fingerprint: identity(manifest),
|
|
65
|
+
downloadBytes,
|
|
66
|
+
requiredFreeBytes: downloadBytes + 512 * 1024 ** 2,
|
|
67
|
+
artifacts: manifest.artifacts,
|
|
68
|
+
};
|
|
44
69
|
}
|
|
45
|
-
async function syncDirectory(path) { const file = await open(path, constants.O_RDONLY); try {
|
|
46
|
-
await file.sync();
|
|
47
|
-
}
|
|
48
|
-
finally {
|
|
49
|
-
await file.close();
|
|
50
|
-
} }
|
|
51
70
|
const uid = () => process.getuid?.();
|
|
52
71
|
const packageName = /^[a-f0-9]{64}$/, linkHolder = /^(?:[a-f0-9]{64}|\.(?:model-install|gc)-[-a-f0-9]{36})$/;
|
|
53
72
|
const missing = (error) => error.code === 'ENOENT';
|
|
54
|
-
async function exists(path) {
|
|
55
|
-
|
|
56
|
-
|
|
73
|
+
async function exists(path) {
|
|
74
|
+
try {
|
|
75
|
+
await lstat(path);
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
if (missing(error))
|
|
80
|
+
return false;
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async function privateDirectory(path) {
|
|
85
|
+
const stat = await lstat(path);
|
|
86
|
+
return stat.isDirectory() && !stat.isSymbolicLink() && stat.uid === uid() && (stat.mode & 0o077) === 0;
|
|
57
87
|
}
|
|
58
|
-
catch (error) {
|
|
59
|
-
if (missing(error))
|
|
60
|
-
return false;
|
|
61
|
-
throw error;
|
|
62
|
-
} }
|
|
63
|
-
async function privateDirectory(path) { const stat = await lstat(path); return stat.isDirectory() && !stat.isSymbolicLink() && stat.uid === uid() && (stat.mode & 0o077) === 0; }
|
|
64
88
|
/** A shared artifact inode is trusted only when every one of its links is a top-level file of a private package,
|
|
65
89
|
* stage or GC directory of this cache: nothing outside the private cache can reach (or later rewrite) it. */
|
|
66
90
|
async function linksAccounted(root, stat) {
|
|
67
91
|
if (stat.nlink > 64)
|
|
68
92
|
return false;
|
|
69
|
-
const holders = (await readdir(root)).filter(name => linkHolder.test(name));
|
|
93
|
+
const holders = (await readdir(root)).filter((name) => linkHolder.test(name));
|
|
70
94
|
if (holders.length > 256)
|
|
71
95
|
return false;
|
|
72
96
|
let count = 0;
|
|
@@ -74,7 +98,7 @@ async function linksAccounted(root, stat) {
|
|
|
74
98
|
const dir = join(root, name);
|
|
75
99
|
let entries;
|
|
76
100
|
try {
|
|
77
|
-
if (!await privateDirectory(dir))
|
|
101
|
+
if (!(await privateDirectory(dir)))
|
|
78
102
|
continue;
|
|
79
103
|
entries = await readdir(dir);
|
|
80
104
|
}
|
|
@@ -105,8 +129,13 @@ async function verifyArtifact(path, artifact, root) {
|
|
|
105
129
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
106
130
|
try {
|
|
107
131
|
const before = await file.stat();
|
|
108
|
-
if (!before.isFile() ||
|
|
109
|
-
|
|
132
|
+
if (!before.isFile() ||
|
|
133
|
+
before.nlink < 1 ||
|
|
134
|
+
(before.nlink !== 1 && !(root && (await linksAccounted(root, before)))) ||
|
|
135
|
+
before.size !== artifact.bytes ||
|
|
136
|
+
(before.mode & 0o077) !== 0 ||
|
|
137
|
+
(artifact.executable && (before.mode & 0o100) === 0))
|
|
138
|
+
throw new SpongeError('integrity');
|
|
110
139
|
const hash = createHash('sha256'), buffer = Buffer.alloc(65536);
|
|
111
140
|
let bytes = 0;
|
|
112
141
|
for (;;) {
|
|
@@ -115,12 +144,15 @@ async function verifyArtifact(path, artifact, root) {
|
|
|
115
144
|
break;
|
|
116
145
|
bytes += read.bytesRead;
|
|
117
146
|
if (bytes > artifact.bytes)
|
|
118
|
-
throw new
|
|
147
|
+
throw new SpongeError('integrity');
|
|
119
148
|
hash.update(buffer.subarray(0, read.bytesRead));
|
|
120
149
|
}
|
|
121
150
|
const after = await file.stat();
|
|
122
|
-
if (bytes !== artifact.bytes ||
|
|
123
|
-
|
|
151
|
+
if (bytes !== artifact.bytes ||
|
|
152
|
+
hash.digest('hex') !== artifact.sha256 ||
|
|
153
|
+
before.mtimeMs !== after.mtimeMs ||
|
|
154
|
+
before.nlink !== after.nlink)
|
|
155
|
+
throw new SpongeError('integrity');
|
|
124
156
|
}
|
|
125
157
|
finally {
|
|
126
158
|
await file.close();
|
|
@@ -134,21 +166,21 @@ async function writeStaged(path, artifact, chunks, signal) {
|
|
|
134
166
|
try {
|
|
135
167
|
for await (const chunk of chunks) {
|
|
136
168
|
if (signal?.aborted)
|
|
137
|
-
throw new
|
|
169
|
+
throw new SpongeError('cancelled');
|
|
138
170
|
bytes += chunk.length;
|
|
139
171
|
if (bytes > artifact.bytes)
|
|
140
|
-
throw new
|
|
172
|
+
throw new SpongeError('integrity');
|
|
141
173
|
hash.update(chunk);
|
|
142
174
|
let offset = 0;
|
|
143
175
|
while (offset < chunk.length) {
|
|
144
176
|
const result = await file.write(chunk, offset, chunk.length - offset);
|
|
145
177
|
if (!result.bytesWritten)
|
|
146
|
-
throw new
|
|
178
|
+
throw new SpongeError('unavailable');
|
|
147
179
|
offset += result.bytesWritten;
|
|
148
180
|
}
|
|
149
181
|
}
|
|
150
182
|
if (bytes !== artifact.bytes || hash.digest('hex') !== artifact.sha256)
|
|
151
|
-
throw new
|
|
183
|
+
throw new SpongeError('integrity');
|
|
152
184
|
await file.chmod(artifact.executable ? 0o700 : 0o600);
|
|
153
185
|
await file.sync();
|
|
154
186
|
}
|
|
@@ -159,12 +191,19 @@ async function writeStaged(path, artifact, chunks, signal) {
|
|
|
159
191
|
/** Same name, size and mode class in another installed package that no setup or recovery currently holds. */
|
|
160
192
|
async function reuseCandidates(root, artifact, exclude) {
|
|
161
193
|
const result = [];
|
|
162
|
-
for (const name of (await readdir(root)).filter(n => packageName.test(n) && n !== exclude).sort()) {
|
|
194
|
+
for (const name of (await readdir(root)).filter((n) => packageName.test(n) && n !== exclude).sort()) {
|
|
163
195
|
try {
|
|
164
|
-
if (await exists(join(root, `.${name}.lock`))
|
|
196
|
+
if ((await exists(join(root, `.${name}.lock`))) ||
|
|
197
|
+
(await exists(join(root, `.${name}.recovery-lock`))) ||
|
|
198
|
+
!(await privateDirectory(join(root, name))))
|
|
165
199
|
continue;
|
|
166
200
|
const s = await lstat(join(root, name, artifact.name));
|
|
167
|
-
if (s.isFile() &&
|
|
201
|
+
if (s.isFile() &&
|
|
202
|
+
!s.isSymbolicLink() &&
|
|
203
|
+
s.uid === uid() &&
|
|
204
|
+
s.size === artifact.bytes &&
|
|
205
|
+
(s.mode & 0o077) === 0 &&
|
|
206
|
+
!!artifact.executable === ((s.mode & 0o100) !== 0))
|
|
168
207
|
result.push(join(root, name, artifact.name));
|
|
169
208
|
}
|
|
170
209
|
catch (error) {
|
|
@@ -174,14 +213,15 @@ async function reuseCandidates(root, artifact, exclude) {
|
|
|
174
213
|
}
|
|
175
214
|
return result;
|
|
176
215
|
}
|
|
177
|
-
/** Upgrade reuse:
|
|
216
|
+
/** Upgrade reuse: another installed package (an earlier URL-bound fingerprint, or a partly changed release) may carry
|
|
217
|
+
* byte-identical artifacts. A candidate
|
|
178
218
|
* is never trusted: it is opened without following links, hard-linked (same inode as the opened file, or undone) or
|
|
179
219
|
* else copied from that descriptor, and the staged file is re-hashed against THIS manifest. Any doubt downloads. */
|
|
180
220
|
async function reuseArtifact(root, stage, artifact, exclude, signal) {
|
|
181
221
|
const target = join(stage, artifact.name), mode = artifact.executable ? 0o700 : 0o600;
|
|
182
222
|
for (const candidate of await reuseCandidates(root, artifact, exclude)) {
|
|
183
223
|
if (signal?.aborted)
|
|
184
|
-
throw new
|
|
224
|
+
throw new SpongeError('cancelled');
|
|
185
225
|
let source;
|
|
186
226
|
try {
|
|
187
227
|
source = await open(candidate, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
@@ -191,7 +231,11 @@ async function reuseArtifact(root, stage, artifact, exclude, signal) {
|
|
|
191
231
|
}
|
|
192
232
|
try {
|
|
193
233
|
const s = await source.stat();
|
|
194
|
-
if (!s.isFile() ||
|
|
234
|
+
if (!s.isFile() ||
|
|
235
|
+
s.uid !== uid() ||
|
|
236
|
+
s.size !== artifact.bytes ||
|
|
237
|
+
(s.mode & 0o077) !== 0 ||
|
|
238
|
+
!!artifact.executable !== ((s.mode & 0o100) !== 0))
|
|
195
239
|
continue;
|
|
196
240
|
if ((s.mode & 0o777) === mode) {
|
|
197
241
|
// A link to anything but the opened inode, or bytes failing this manifest, is undone and the candidate skipped.
|
|
@@ -206,7 +250,7 @@ async function reuseArtifact(root, stage, artifact, exclude, signal) {
|
|
|
206
250
|
}
|
|
207
251
|
}
|
|
208
252
|
catch (error) {
|
|
209
|
-
if (linked && !(error instanceof
|
|
253
|
+
if (linked && !(error instanceof SpongeError))
|
|
210
254
|
throw error;
|
|
211
255
|
}
|
|
212
256
|
if (linked) {
|
|
@@ -224,7 +268,7 @@ async function reuseArtifact(root, stage, artifact, exclude, signal) {
|
|
|
224
268
|
}
|
|
225
269
|
catch (error) {
|
|
226
270
|
await rm(target, { force: true });
|
|
227
|
-
if (error
|
|
271
|
+
if (codeOf(error) === 'cancelled')
|
|
228
272
|
throw error;
|
|
229
273
|
}
|
|
230
274
|
}
|
|
@@ -245,94 +289,133 @@ export async function inspectModelPackage(cacheRoot, manifest) {
|
|
|
245
289
|
}
|
|
246
290
|
return paths;
|
|
247
291
|
}
|
|
248
|
-
const pending = () => {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
292
|
+
const pending = () => {
|
|
293
|
+
throw new SpongeError('cleanup-required');
|
|
294
|
+
};
|
|
295
|
+
async function sameFile(a, b) {
|
|
296
|
+
try {
|
|
297
|
+
const [x, y] = await Promise.all([lstat(a), lstat(b)]);
|
|
298
|
+
if (!x.isFile() || !y.isFile() || x.isSymbolicLink() || y.isSymbolicLink() || x.ino !== y.ino || x.dev !== y.dev)
|
|
299
|
+
pending();
|
|
300
|
+
}
|
|
301
|
+
catch {
|
|
302
|
+
return pending();
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
async function readOwner(path, fingerprint) {
|
|
306
|
+
const owner = (await readJournal(path));
|
|
307
|
+
if (owner.schemaVersion !== 1 ||
|
|
308
|
+
owner.fingerprint !== fingerprint ||
|
|
309
|
+
!/^[-a-f0-9]{36}$/.test(owner.incarnation) ||
|
|
310
|
+
owner.stage !== '.model-install-' + owner.incarnation ||
|
|
311
|
+
!owner.identity ||
|
|
312
|
+
!Number.isSafeInteger(owner.identity.pid) ||
|
|
313
|
+
owner.identity.pid < 1 ||
|
|
314
|
+
owner.identity.uid !== process.getuid?.() ||
|
|
315
|
+
typeof owner.identity.started !== 'string' ||
|
|
316
|
+
!owner.identity.started.trim())
|
|
252
317
|
pending();
|
|
318
|
+
return owner;
|
|
253
319
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
320
|
+
async function cleanupStage(root, owner, file) {
|
|
321
|
+
const stage = join(root, owner.stage);
|
|
322
|
+
let stat;
|
|
323
|
+
try {
|
|
324
|
+
stat = await lstat(stage);
|
|
325
|
+
}
|
|
326
|
+
catch (error) {
|
|
327
|
+
if (error.code === 'ENOENT')
|
|
328
|
+
return;
|
|
329
|
+
throw error;
|
|
330
|
+
}
|
|
331
|
+
if (!stat.isDirectory() ||
|
|
332
|
+
stat.isSymbolicLink() ||
|
|
333
|
+
stat.uid !== process.getuid?.() ||
|
|
334
|
+
(stat.mode & 0o077) !== 0 ||
|
|
335
|
+
(await realpath(stage)) !== stage)
|
|
336
|
+
pending();
|
|
337
|
+
const entries = await readdir(stage);
|
|
338
|
+
if (entries.length) {
|
|
339
|
+
await sameFile(file, join(stage, 'ownership.json'));
|
|
340
|
+
}
|
|
341
|
+
await rm(stage, { recursive: true });
|
|
342
|
+
await syncDirectoryAsync(root);
|
|
261
343
|
}
|
|
262
|
-
catch (error) {
|
|
263
|
-
if (error.code === 'ENOENT')
|
|
264
|
-
return;
|
|
265
|
-
throw error;
|
|
266
|
-
} if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid?.() || (stat.mode & 0o077) !== 0 || await realpath(stage) !== stage)
|
|
267
|
-
pending(); const entries = await readdir(stage); if (entries.length) {
|
|
268
|
-
await sameFile(file, join(stage, 'ownership.json'));
|
|
269
|
-
} await rm(stage, { recursive: true }); await syncDirectory(root); }
|
|
270
344
|
/** Explicit recovery only: an absent or different process birth identity proves the downloader is gone.
|
|
271
345
|
* Downloading is in-process fetch; no native child can outlive this owner. A missing or
|
|
272
|
-
* foreign stage receipt remains pending, as does
|
|
346
|
+
* foreign stage receipt remains pending, as does a recovery guard held by a live or unknown recoverer. The guard names
|
|
347
|
+
* the recovering process, so a recovery that itself died is taken over on exact proof of that death. */
|
|
348
|
+
async function recoveryGuard(guard) {
|
|
349
|
+
try {
|
|
350
|
+
return await acquireIdentityLock(guard);
|
|
351
|
+
}
|
|
352
|
+
catch (error) {
|
|
353
|
+
const code = codeOf(error);
|
|
354
|
+
if (code === 'owner-busy')
|
|
355
|
+
return pending();
|
|
356
|
+
if (code === 'unavailable')
|
|
357
|
+
throw new SpongeError('unavailable');
|
|
358
|
+
throw error;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
273
361
|
export async function recoverModelPackage(cacheRoot, manifest) {
|
|
274
|
-
const plan = modelSetupPlan(manifest), root = await privateRoot(cacheRoot), lock = join(root, `.${plan.fingerprint}.lock`);
|
|
362
|
+
const plan = modelSetupPlan(manifest), root = await privateRoot(cacheRoot), lock = join(root, `.${plan.fingerprint}.lock`), guard = join(root, `.${plan.fingerprint}.recovery-lock`);
|
|
275
363
|
let owner;
|
|
364
|
+
// A recovery that died after unlinking the owner lock leaves only its guard: clear it when its holder is provably dead.
|
|
276
365
|
try {
|
|
277
366
|
owner = await readOwner(lock, plan.fingerprint);
|
|
278
367
|
}
|
|
279
368
|
catch (error) {
|
|
280
|
-
if (error.code === 'ENOENT')
|
|
369
|
+
if (error.code === 'ENOENT') {
|
|
370
|
+
if (await exists(guard))
|
|
371
|
+
await (await recoveryGuard(guard)).release();
|
|
281
372
|
return { recovered: false };
|
|
373
|
+
}
|
|
282
374
|
return pending();
|
|
283
375
|
}
|
|
284
376
|
try {
|
|
285
377
|
if (await identityAlive(owner.identity))
|
|
286
|
-
throw new
|
|
378
|
+
throw new SpongeError('owner-busy');
|
|
287
379
|
}
|
|
288
380
|
catch (error) {
|
|
289
|
-
if (error
|
|
381
|
+
if (codeOf(error) === 'owner-busy')
|
|
290
382
|
throw error;
|
|
291
383
|
return pending();
|
|
292
384
|
}
|
|
293
|
-
const file = join(root, `.model-owner-${owner.incarnation}.json`)
|
|
385
|
+
const file = join(root, `.model-owner-${owner.incarnation}.json`);
|
|
294
386
|
await sameFile(file, lock);
|
|
295
|
-
|
|
296
|
-
await link(file, guard);
|
|
297
|
-
await syncDirectory(root);
|
|
298
|
-
}
|
|
299
|
-
catch (error) {
|
|
300
|
-
if (error.code === 'EEXIST')
|
|
301
|
-
return pending();
|
|
302
|
-
throw error;
|
|
303
|
-
}
|
|
387
|
+
const held = await recoveryGuard(guard);
|
|
304
388
|
try {
|
|
305
389
|
await sameFile(file, lock);
|
|
306
390
|
if (await identityAlive(owner.identity))
|
|
307
|
-
throw new
|
|
391
|
+
throw new SpongeError('owner-busy');
|
|
308
392
|
await cleanupStage(root, owner, file);
|
|
309
393
|
await sameFile(file, lock);
|
|
310
394
|
await unlink(lock);
|
|
311
|
-
await
|
|
395
|
+
await syncDirectoryAsync(root);
|
|
312
396
|
return { recovered: true };
|
|
313
397
|
}
|
|
314
398
|
finally {
|
|
315
|
-
await
|
|
316
|
-
await unlink(guard);
|
|
317
|
-
await syncDirectory(root);
|
|
399
|
+
await held.release();
|
|
318
400
|
try {
|
|
319
401
|
await lstat(lock);
|
|
320
402
|
}
|
|
321
403
|
catch (error) {
|
|
322
404
|
if (error.code === 'ENOENT') {
|
|
323
405
|
await unlink(file);
|
|
324
|
-
await
|
|
406
|
+
await syncDirectoryAsync(root);
|
|
325
407
|
}
|
|
326
408
|
else
|
|
327
409
|
throw error;
|
|
328
410
|
}
|
|
329
411
|
}
|
|
330
412
|
}
|
|
331
|
-
/** Explicit setup only. Never called by install hooks or by inference fallback
|
|
413
|
+
/** Explicit setup only. Never called by install hooks or by inference fallback, except `reuseOnly`: that adopts an
|
|
414
|
+
* installed package from verified local artifacts under the same setup lock and never reaches the network. */
|
|
332
415
|
export async function provisionModelPackage(cacheRoot, manifest, options = {}) {
|
|
333
416
|
const plan = modelSetupPlan(manifest), root = await privateRoot(cacheRoot), destination = join(root, plan.fingerprint);
|
|
334
417
|
if (options.signal?.aborted)
|
|
335
|
-
throw new
|
|
418
|
+
throw new SpongeError('cancelled');
|
|
336
419
|
try {
|
|
337
420
|
await lstat(destination);
|
|
338
421
|
return await inspectModelPackage(root, manifest);
|
|
@@ -341,25 +424,36 @@ export async function provisionModelPackage(cacheRoot, manifest, options = {}) {
|
|
|
341
424
|
if (error.code !== 'ENOENT')
|
|
342
425
|
throw error;
|
|
343
426
|
}
|
|
427
|
+
// Nothing to adopt takes no lock and writes nothing.
|
|
428
|
+
if (options.reuseOnly)
|
|
429
|
+
for (const artifact of manifest.artifacts)
|
|
430
|
+
if (!(await reuseCandidates(root, artifact, plan.fingerprint)).length)
|
|
431
|
+
throw new SpongeError('unavailable');
|
|
344
432
|
const lock = join(root, `.${plan.fingerprint}.lock`), incarnation = randomUUID(), ownerFile = join(root, `.model-owner-${incarnation}.json`);
|
|
345
433
|
const identity = await processIdentity(process.pid);
|
|
346
434
|
if (!identity)
|
|
347
435
|
pending();
|
|
348
|
-
const owner = {
|
|
436
|
+
const owner = {
|
|
437
|
+
schemaVersion: 1,
|
|
438
|
+
incarnation,
|
|
439
|
+
fingerprint: plan.fingerprint,
|
|
440
|
+
stage: '.model-install-' + incarnation,
|
|
441
|
+
identity: identity,
|
|
442
|
+
};
|
|
349
443
|
await writeJournal(ownerFile, owner);
|
|
350
444
|
let linked = false;
|
|
351
445
|
try {
|
|
352
446
|
await link(ownerFile, lock);
|
|
353
447
|
linked = true;
|
|
354
|
-
await
|
|
448
|
+
await syncDirectoryAsync(root);
|
|
355
449
|
}
|
|
356
450
|
catch (error) {
|
|
357
451
|
if (linked)
|
|
358
452
|
return pending();
|
|
359
453
|
await unlink(ownerFile);
|
|
360
|
-
await
|
|
454
|
+
await syncDirectoryAsync(root);
|
|
361
455
|
if (error.code === 'EEXIST')
|
|
362
|
-
throw new
|
|
456
|
+
throw new SpongeError('owner-busy');
|
|
363
457
|
throw error;
|
|
364
458
|
}
|
|
365
459
|
let stage;
|
|
@@ -371,38 +465,42 @@ export async function provisionModelPackage(cacheRoot, manifest, options = {}) {
|
|
|
371
465
|
needed += artifact.bytes;
|
|
372
466
|
const disk = await statfs(root);
|
|
373
467
|
if (disk.bavail * disk.bsize < needed)
|
|
374
|
-
throw new
|
|
468
|
+
throw new SpongeError('limit-exceeded');
|
|
375
469
|
stage = join(root, owner.stage);
|
|
376
470
|
await mkdir(stage, { mode: 0o700 });
|
|
377
471
|
await chmod(stage, 0o700);
|
|
378
472
|
await link(ownerFile, join(stage, 'ownership.json'));
|
|
379
|
-
await
|
|
380
|
-
await
|
|
473
|
+
await syncDirectoryAsync(stage);
|
|
474
|
+
await syncDirectoryAsync(root);
|
|
381
475
|
for (const artifact of manifest.artifacts) {
|
|
382
476
|
if (await reuseArtifact(root, stage, artifact, plan.fingerprint, options.signal))
|
|
383
477
|
continue;
|
|
384
|
-
|
|
478
|
+
if (options.reuseOnly)
|
|
479
|
+
throw new SpongeError('unavailable');
|
|
480
|
+
const signal = options.signal
|
|
481
|
+
? AbortSignal.any([options.signal, AbortSignal.timeout(600000)])
|
|
482
|
+
: AbortSignal.timeout(600000);
|
|
385
483
|
const response = await (options.fetch ?? fetch)(artifact.url, { signal });
|
|
386
484
|
if (!response.ok || !response.body || (response.url && new URL(response.url).protocol !== 'https:'))
|
|
387
|
-
throw new
|
|
485
|
+
throw new SpongeError('unavailable');
|
|
388
486
|
const declared = response.headers.get('content-length');
|
|
389
487
|
if (declared !== null && Number(declared) !== artifact.bytes) {
|
|
390
488
|
await response.body.cancel();
|
|
391
|
-
throw new
|
|
489
|
+
throw new SpongeError('integrity');
|
|
392
490
|
}
|
|
393
491
|
await writeStaged(join(stage, artifact.name), artifact, response.body, signal);
|
|
394
492
|
}
|
|
395
493
|
if (options.signal?.aborted)
|
|
396
|
-
throw new
|
|
397
|
-
await
|
|
494
|
+
throw new SpongeError('cancelled');
|
|
495
|
+
await syncDirectoryAsync(stage);
|
|
398
496
|
await rename(stage, destination);
|
|
399
497
|
stage = undefined;
|
|
400
|
-
await
|
|
498
|
+
await syncDirectoryAsync(root);
|
|
401
499
|
return await inspectModelPackage(root, manifest);
|
|
402
500
|
}
|
|
403
501
|
catch (error) {
|
|
404
502
|
if (options.signal?.aborted)
|
|
405
|
-
throw new
|
|
503
|
+
throw new SpongeError('cancelled');
|
|
406
504
|
throw error;
|
|
407
505
|
}
|
|
408
506
|
finally {
|
|
@@ -411,6 +509,39 @@ export async function provisionModelPackage(cacheRoot, manifest, options = {}) {
|
|
|
411
509
|
await sameFile(ownerFile, lock);
|
|
412
510
|
await unlink(lock);
|
|
413
511
|
await unlink(ownerFile);
|
|
414
|
-
await
|
|
512
|
+
await syncDirectoryAsync(root);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
const absent = (error) => error.code === 'ENOENT';
|
|
516
|
+
/** The installed package for this manifest, by content, or null when none is. A package installed under an earlier
|
|
517
|
+
* URL-bound fingerprint is adopted once: its verified artifacts are hard-linked into the content-named directory under
|
|
518
|
+
* the setup lock (no network; the old directory stays intact for any process holding its paths until runtime GC
|
|
519
|
+
* supersedes it). Anything short of a fully verified adoption is reported as not installed, as before. */
|
|
520
|
+
export async function inspectInstalledModelPackage(cacheRoot, manifest, options = {}) {
|
|
521
|
+
try {
|
|
522
|
+
return await inspectModelPackage(cacheRoot, manifest);
|
|
415
523
|
}
|
|
524
|
+
catch (error) {
|
|
525
|
+
if (!absent(error))
|
|
526
|
+
throw error;
|
|
527
|
+
}
|
|
528
|
+
try {
|
|
529
|
+
return await provisionModelPackage(cacheRoot, manifest, { reuseOnly: true });
|
|
530
|
+
}
|
|
531
|
+
catch (error) {
|
|
532
|
+
if (codeOf(error) !== 'owner-busy')
|
|
533
|
+
return null;
|
|
534
|
+
}
|
|
535
|
+
// A concurrent adoption or setup of this package is linking or downloading: briefly wait for its verified result.
|
|
536
|
+
for (let i = 0; i < 20; i++) {
|
|
537
|
+
await (options.pause ?? ((ms) => new Promise((r) => setTimeout(r, ms))))(100);
|
|
538
|
+
try {
|
|
539
|
+
return await inspectModelPackage(cacheRoot, manifest);
|
|
540
|
+
}
|
|
541
|
+
catch (error) {
|
|
542
|
+
if (!absent(error))
|
|
543
|
+
throw error;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
return null;
|
|
416
547
|
}
|
|
@@ -2,7 +2,7 @@ import type { ModelPackage } from './artifacts.js';
|
|
|
2
2
|
export type NativeKind = 'asr' | 'embedding';
|
|
3
3
|
export type NativeRole = 'asr-helper' | 'embedding-helper' | 'embedding-library' | 'notices';
|
|
4
4
|
export declare const nativeAssetNames: Record<NativeRole, string>;
|
|
5
|
-
|
|
5
|
+
interface NativeArtifact {
|
|
6
6
|
role: NativeRole;
|
|
7
7
|
name: string;
|
|
8
8
|
bytes: number;
|
|
@@ -23,13 +23,10 @@ export interface NativeCatalog {
|
|
|
23
23
|
clangVersion: string;
|
|
24
24
|
};
|
|
25
25
|
}
|
|
26
|
-
export declare class NativeCatalogError extends Error {
|
|
27
|
-
readonly code: 'runtime-missing' | 'unsupported' | 'integrity';
|
|
28
|
-
constructor(code: 'runtime-missing' | 'unsupported' | 'integrity');
|
|
29
|
-
}
|
|
30
26
|
/** Package-embedded release hashes are authority; local probe helper hashes never enter this catalog. */
|
|
31
27
|
export declare function parseNativeCatalog(value: unknown, expectedVersion: string): NativeCatalog;
|
|
32
28
|
export declare function nativePlatformSupported(platform?: NodeJS.Platform, architecture?: NodeJS.Architecture, kernel?: string): boolean;
|
|
33
29
|
export declare function nativeCatalogPath(): string;
|
|
34
30
|
export declare function loadNativeCatalog(expectedVersion: string, path?: string): Promise<NativeCatalog>;
|
|
35
31
|
export declare function nativeModelPackage(catalog: NativeCatalog, kind: NativeKind): ModelPackage;
|
|
32
|
+
export {};
|