@hraness/sponge 0.2.5 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +23 -10
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +39 -17
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +10 -46
  64. package/dist/src/digest/editorial.js +73 -59
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +53 -32
  67. package/dist/src/digest/pipeline.js +357 -125
  68. package/dist/src/digest/schema.d.ts +44 -17
  69. package/dist/src/digest/schema.js +132 -44
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +31 -24
  82. package/dist/src/importers/jungle-digest.js +217 -115
  83. package/dist/src/importers/jungle-store.d.ts +20 -11
  84. package/dist/src/importers/jungle-store.js +136 -41
  85. package/dist/src/importers/jungle.d.ts +32 -16
  86. package/dist/src/importers/jungle.js +145 -32
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +108 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -77
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -4,18 +4,18 @@ import { dirname, join } from 'node:path';
4
4
  import { createHash, randomUUID } from 'node:crypto';
5
5
  import { processIdentity, identityAlive } from '../worker/identity.js';
6
6
  import { readJournal, writeJournal } from '../worker/journal.js';
7
+ import { acquireIdentityLock } from '../worker/lock.js';
8
+ import { syncDirectoryAsync } from '../util/fsync.js';
7
9
  import { privateRoot } from '../acquisition/files.js';
8
- export class ModelSetupError extends Error {
9
- code;
10
- constructor(code) {
11
- super(code);
12
- this.code = code;
13
- }
14
- }
10
+ import { SpongeError, codeOf } from '../errors.js';
15
11
  const maximumBytes = 2 * 1024 ** 3;
16
12
  function validate(manifest) {
17
- if (!/^[a-z0-9][a-z0-9.-]{0,127}$/.test(manifest.id) || !manifest.license || manifest.license.length > 256 || !manifest.artifacts.length || manifest.artifacts.length > 16)
18
- throw new ModelSetupError('invalid-input');
13
+ if (!/^[a-z0-9][a-z0-9.-]{0,127}$/.test(manifest.id) ||
14
+ !manifest.license ||
15
+ manifest.license.length > 256 ||
16
+ !manifest.artifacts.length ||
17
+ manifest.artifacts.length > 16)
18
+ throw new SpongeError('invalid-input');
19
19
  const names = new Set();
20
20
  let bytes = 0;
21
21
  for (const artifact of manifest.artifacts) {
@@ -24,49 +24,73 @@ function validate(manifest) {
24
24
  url = new URL(artifact.url);
25
25
  }
26
26
  catch {
27
- throw new ModelSetupError('invalid-input');
27
+ throw new SpongeError('invalid-input');
28
28
  }
29
- if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(artifact.name) || names.has(artifact.name) || url.protocol !== 'https:' || url.username || url.password || !/^[a-f0-9]{64}$/.test(artifact.sha256) || !Number.isSafeInteger(artifact.bytes) || artifact.bytes < 1 || artifact.bytes > maximumBytes)
30
- throw new ModelSetupError('invalid-input');
29
+ if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(artifact.name) ||
30
+ names.has(artifact.name) ||
31
+ url.protocol !== 'https:' ||
32
+ url.username ||
33
+ url.password ||
34
+ !/^[a-f0-9]{64}$/.test(artifact.sha256) ||
35
+ !Number.isSafeInteger(artifact.bytes) ||
36
+ artifact.bytes < 1 ||
37
+ artifact.bytes > maximumBytes)
38
+ throw new SpongeError('invalid-input');
31
39
  names.add(artifact.name);
32
40
  bytes += artifact.bytes;
33
41
  }
34
42
  if (bytes > maximumBytes)
35
- throw new ModelSetupError('limit-exceeded');
43
+ throw new SpongeError('limit-exceeded');
36
44
  }
45
+ /** Installed-package identity is content only: artifact names, bytes, sha256 and executable bit. The package id,
46
+ * license text and per-release download URLs are metadata, so a release that ships byte-identical artifacts keeps
47
+ * the installed directory. The domain tag keeps these names disjoint from the earlier URL-bound fingerprints. */
37
48
  function identity(manifest) {
38
- return createHash('sha256').update(JSON.stringify({ id: manifest.id, license: manifest.license, artifacts: manifest.artifacts.map(a => ({ name: a.name, url: a.url, sha256: a.sha256, bytes: a.bytes, executable: !!a.executable })).sort((a, b) => a.name.localeCompare(b.name)) })).digest('hex');
49
+ return createHash('sha256')
50
+ .update(JSON.stringify({
51
+ identity: 'sponge-model-content-v1',
52
+ artifacts: manifest.artifacts
53
+ .map((a) => ({ name: a.name, sha256: a.sha256, bytes: a.bytes, executable: !!a.executable }))
54
+ .sort((a, b) => a.name.localeCompare(b.name)),
55
+ }))
56
+ .digest('hex');
39
57
  }
40
58
  export function modelSetupPlan(manifest) {
41
59
  validate(manifest);
42
60
  const downloadBytes = manifest.artifacts.reduce((n, a) => n + a.bytes, 0);
43
- return { id: manifest.id, license: manifest.license, fingerprint: identity(manifest), downloadBytes, requiredFreeBytes: downloadBytes + 512 * 1024 ** 2, artifacts: manifest.artifacts };
61
+ return {
62
+ id: manifest.id,
63
+ license: manifest.license,
64
+ fingerprint: identity(manifest),
65
+ downloadBytes,
66
+ requiredFreeBytes: downloadBytes + 512 * 1024 ** 2,
67
+ artifacts: manifest.artifacts,
68
+ };
44
69
  }
45
- async function syncDirectory(path) { const file = await open(path, constants.O_RDONLY); try {
46
- await file.sync();
47
- }
48
- finally {
49
- await file.close();
50
- } }
51
70
  const uid = () => process.getuid?.();
52
71
  const packageName = /^[a-f0-9]{64}$/, linkHolder = /^(?:[a-f0-9]{64}|\.(?:model-install|gc)-[-a-f0-9]{36})$/;
53
72
  const missing = (error) => error.code === 'ENOENT';
54
- async function exists(path) { try {
55
- await lstat(path);
56
- return true;
73
+ async function exists(path) {
74
+ try {
75
+ await lstat(path);
76
+ return true;
77
+ }
78
+ catch (error) {
79
+ if (missing(error))
80
+ return false;
81
+ throw error;
82
+ }
83
+ }
84
+ async function privateDirectory(path) {
85
+ const stat = await lstat(path);
86
+ return stat.isDirectory() && !stat.isSymbolicLink() && stat.uid === uid() && (stat.mode & 0o077) === 0;
57
87
  }
58
- catch (error) {
59
- if (missing(error))
60
- return false;
61
- throw error;
62
- } }
63
- async function privateDirectory(path) { const stat = await lstat(path); return stat.isDirectory() && !stat.isSymbolicLink() && stat.uid === uid() && (stat.mode & 0o077) === 0; }
64
88
  /** A shared artifact inode is trusted only when every one of its links is a top-level file of a private package,
65
89
  * stage or GC directory of this cache: nothing outside the private cache can reach (or later rewrite) it. */
66
90
  async function linksAccounted(root, stat) {
67
91
  if (stat.nlink > 64)
68
92
  return false;
69
- const holders = (await readdir(root)).filter(name => linkHolder.test(name));
93
+ const holders = (await readdir(root)).filter((name) => linkHolder.test(name));
70
94
  if (holders.length > 256)
71
95
  return false;
72
96
  let count = 0;
@@ -74,7 +98,7 @@ async function linksAccounted(root, stat) {
74
98
  const dir = join(root, name);
75
99
  let entries;
76
100
  try {
77
- if (!await privateDirectory(dir))
101
+ if (!(await privateDirectory(dir)))
78
102
  continue;
79
103
  entries = await readdir(dir);
80
104
  }
@@ -105,8 +129,13 @@ async function verifyArtifact(path, artifact, root) {
105
129
  const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
106
130
  try {
107
131
  const before = await file.stat();
108
- if (!before.isFile() || before.nlink < 1 || before.nlink !== 1 && !(root && await linksAccounted(root, before)) || before.size !== artifact.bytes || (before.mode & 0o077) !== 0 || (artifact.executable && (before.mode & 0o100) === 0))
109
- throw new ModelSetupError('integrity');
132
+ if (!before.isFile() ||
133
+ before.nlink < 1 ||
134
+ (before.nlink !== 1 && !(root && (await linksAccounted(root, before)))) ||
135
+ before.size !== artifact.bytes ||
136
+ (before.mode & 0o077) !== 0 ||
137
+ (artifact.executable && (before.mode & 0o100) === 0))
138
+ throw new SpongeError('integrity');
110
139
  const hash = createHash('sha256'), buffer = Buffer.alloc(65536);
111
140
  let bytes = 0;
112
141
  for (;;) {
@@ -115,12 +144,15 @@ async function verifyArtifact(path, artifact, root) {
115
144
  break;
116
145
  bytes += read.bytesRead;
117
146
  if (bytes > artifact.bytes)
118
- throw new ModelSetupError('integrity');
147
+ throw new SpongeError('integrity');
119
148
  hash.update(buffer.subarray(0, read.bytesRead));
120
149
  }
121
150
  const after = await file.stat();
122
- if (bytes !== artifact.bytes || hash.digest('hex') !== artifact.sha256 || before.mtimeMs !== after.mtimeMs || before.nlink !== after.nlink)
123
- throw new ModelSetupError('integrity');
151
+ if (bytes !== artifact.bytes ||
152
+ hash.digest('hex') !== artifact.sha256 ||
153
+ before.mtimeMs !== after.mtimeMs ||
154
+ before.nlink !== after.nlink)
155
+ throw new SpongeError('integrity');
124
156
  }
125
157
  finally {
126
158
  await file.close();
@@ -134,21 +166,21 @@ async function writeStaged(path, artifact, chunks, signal) {
134
166
  try {
135
167
  for await (const chunk of chunks) {
136
168
  if (signal?.aborted)
137
- throw new ModelSetupError('cancelled');
169
+ throw new SpongeError('cancelled');
138
170
  bytes += chunk.length;
139
171
  if (bytes > artifact.bytes)
140
- throw new ModelSetupError('integrity');
172
+ throw new SpongeError('integrity');
141
173
  hash.update(chunk);
142
174
  let offset = 0;
143
175
  while (offset < chunk.length) {
144
176
  const result = await file.write(chunk, offset, chunk.length - offset);
145
177
  if (!result.bytesWritten)
146
- throw new ModelSetupError('unavailable');
178
+ throw new SpongeError('unavailable');
147
179
  offset += result.bytesWritten;
148
180
  }
149
181
  }
150
182
  if (bytes !== artifact.bytes || hash.digest('hex') !== artifact.sha256)
151
- throw new ModelSetupError('integrity');
183
+ throw new SpongeError('integrity');
152
184
  await file.chmod(artifact.executable ? 0o700 : 0o600);
153
185
  await file.sync();
154
186
  }
@@ -159,12 +191,19 @@ async function writeStaged(path, artifact, chunks, signal) {
159
191
  /** Same name, size and mode class in another installed package that no setup or recovery currently holds. */
160
192
  async function reuseCandidates(root, artifact, exclude) {
161
193
  const result = [];
162
- for (const name of (await readdir(root)).filter(n => packageName.test(n) && n !== exclude).sort()) {
194
+ for (const name of (await readdir(root)).filter((n) => packageName.test(n) && n !== exclude).sort()) {
163
195
  try {
164
- if (await exists(join(root, `.${name}.lock`)) || await exists(join(root, `.${name}.recovery-lock`)) || !await privateDirectory(join(root, name)))
196
+ if ((await exists(join(root, `.${name}.lock`))) ||
197
+ (await exists(join(root, `.${name}.recovery-lock`))) ||
198
+ !(await privateDirectory(join(root, name))))
165
199
  continue;
166
200
  const s = await lstat(join(root, name, artifact.name));
167
- if (s.isFile() && !s.isSymbolicLink() && s.uid === uid() && s.size === artifact.bytes && (s.mode & 0o077) === 0 && !!artifact.executable === ((s.mode & 0o100) !== 0))
201
+ if (s.isFile() &&
202
+ !s.isSymbolicLink() &&
203
+ s.uid === uid() &&
204
+ s.size === artifact.bytes &&
205
+ (s.mode & 0o077) === 0 &&
206
+ !!artifact.executable === ((s.mode & 0o100) !== 0))
168
207
  result.push(join(root, name, artifact.name));
169
208
  }
170
209
  catch (error) {
@@ -174,14 +213,15 @@ async function reuseCandidates(root, artifact, exclude) {
174
213
  }
175
214
  return result;
176
215
  }
177
- /** Upgrade reuse: a package whose fingerprint differs only by release URLs carries byte-identical artifacts. A candidate
216
+ /** Upgrade reuse: another installed package (an earlier URL-bound fingerprint, or a partly changed release) may carry
217
+ * byte-identical artifacts. A candidate
178
218
  * is never trusted: it is opened without following links, hard-linked (same inode as the opened file, or undone) or
179
219
  * else copied from that descriptor, and the staged file is re-hashed against THIS manifest. Any doubt downloads. */
180
220
  async function reuseArtifact(root, stage, artifact, exclude, signal) {
181
221
  const target = join(stage, artifact.name), mode = artifact.executable ? 0o700 : 0o600;
182
222
  for (const candidate of await reuseCandidates(root, artifact, exclude)) {
183
223
  if (signal?.aborted)
184
- throw new ModelSetupError('cancelled');
224
+ throw new SpongeError('cancelled');
185
225
  let source;
186
226
  try {
187
227
  source = await open(candidate, constants.O_RDONLY | constants.O_NOFOLLOW);
@@ -191,7 +231,11 @@ async function reuseArtifact(root, stage, artifact, exclude, signal) {
191
231
  }
192
232
  try {
193
233
  const s = await source.stat();
194
- if (!s.isFile() || s.uid !== uid() || s.size !== artifact.bytes || (s.mode & 0o077) !== 0 || !!artifact.executable !== ((s.mode & 0o100) !== 0))
234
+ if (!s.isFile() ||
235
+ s.uid !== uid() ||
236
+ s.size !== artifact.bytes ||
237
+ (s.mode & 0o077) !== 0 ||
238
+ !!artifact.executable !== ((s.mode & 0o100) !== 0))
195
239
  continue;
196
240
  if ((s.mode & 0o777) === mode) {
197
241
  // A link to anything but the opened inode, or bytes failing this manifest, is undone and the candidate skipped.
@@ -206,7 +250,7 @@ async function reuseArtifact(root, stage, artifact, exclude, signal) {
206
250
  }
207
251
  }
208
252
  catch (error) {
209
- if (linked && !(error instanceof ModelSetupError))
253
+ if (linked && !(error instanceof SpongeError))
210
254
  throw error;
211
255
  }
212
256
  if (linked) {
@@ -224,7 +268,7 @@ async function reuseArtifact(root, stage, artifact, exclude, signal) {
224
268
  }
225
269
  catch (error) {
226
270
  await rm(target, { force: true });
227
- if (error instanceof ModelSetupError && error.code === 'cancelled')
271
+ if (codeOf(error) === 'cancelled')
228
272
  throw error;
229
273
  }
230
274
  }
@@ -245,94 +289,133 @@ export async function inspectModelPackage(cacheRoot, manifest) {
245
289
  }
246
290
  return paths;
247
291
  }
248
- const pending = () => { throw new ModelSetupError('cleanup-required'); };
249
- async function sameFile(a, b) { try {
250
- const [x, y] = await Promise.all([lstat(a), lstat(b)]);
251
- if (!x.isFile() || !y.isFile() || x.isSymbolicLink() || y.isSymbolicLink() || x.ino !== y.ino || x.dev !== y.dev)
292
+ const pending = () => {
293
+ throw new SpongeError('cleanup-required');
294
+ };
295
+ async function sameFile(a, b) {
296
+ try {
297
+ const [x, y] = await Promise.all([lstat(a), lstat(b)]);
298
+ if (!x.isFile() || !y.isFile() || x.isSymbolicLink() || y.isSymbolicLink() || x.ino !== y.ino || x.dev !== y.dev)
299
+ pending();
300
+ }
301
+ catch {
302
+ return pending();
303
+ }
304
+ }
305
+ async function readOwner(path, fingerprint) {
306
+ const owner = (await readJournal(path));
307
+ if (owner.schemaVersion !== 1 ||
308
+ owner.fingerprint !== fingerprint ||
309
+ !/^[-a-f0-9]{36}$/.test(owner.incarnation) ||
310
+ owner.stage !== '.model-install-' + owner.incarnation ||
311
+ !owner.identity ||
312
+ !Number.isSafeInteger(owner.identity.pid) ||
313
+ owner.identity.pid < 1 ||
314
+ owner.identity.uid !== process.getuid?.() ||
315
+ typeof owner.identity.started !== 'string' ||
316
+ !owner.identity.started.trim())
252
317
  pending();
318
+ return owner;
253
319
  }
254
- catch {
255
- return pending();
256
- } }
257
- async function readOwner(path, fingerprint) { const owner = await readJournal(path); if (owner.schemaVersion !== 1 || owner.fingerprint !== fingerprint || !/^[-a-f0-9]{36}$/.test(owner.incarnation) || owner.stage !== '.model-install-' + owner.incarnation || !owner.identity || !Number.isSafeInteger(owner.identity.pid) || owner.identity.pid < 1 || owner.identity.uid !== process.getuid?.() || typeof owner.identity.started !== 'string' || !owner.identity.started.trim())
258
- pending(); return owner; }
259
- async function cleanupStage(root, owner, file) { const stage = join(root, owner.stage); let stat; try {
260
- stat = await lstat(stage);
320
+ async function cleanupStage(root, owner, file) {
321
+ const stage = join(root, owner.stage);
322
+ let stat;
323
+ try {
324
+ stat = await lstat(stage);
325
+ }
326
+ catch (error) {
327
+ if (error.code === 'ENOENT')
328
+ return;
329
+ throw error;
330
+ }
331
+ if (!stat.isDirectory() ||
332
+ stat.isSymbolicLink() ||
333
+ stat.uid !== process.getuid?.() ||
334
+ (stat.mode & 0o077) !== 0 ||
335
+ (await realpath(stage)) !== stage)
336
+ pending();
337
+ const entries = await readdir(stage);
338
+ if (entries.length) {
339
+ await sameFile(file, join(stage, 'ownership.json'));
340
+ }
341
+ await rm(stage, { recursive: true });
342
+ await syncDirectoryAsync(root);
261
343
  }
262
- catch (error) {
263
- if (error.code === 'ENOENT')
264
- return;
265
- throw error;
266
- } if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid?.() || (stat.mode & 0o077) !== 0 || await realpath(stage) !== stage)
267
- pending(); const entries = await readdir(stage); if (entries.length) {
268
- await sameFile(file, join(stage, 'ownership.json'));
269
- } await rm(stage, { recursive: true }); await syncDirectory(root); }
270
344
  /** Explicit recovery only: an absent or different process birth identity proves the downloader is gone.
271
345
  * Downloading is in-process fetch; no native child can outlive this owner. A missing or
272
- * foreign stage receipt remains pending, as does an interrupted recovery guard. */
346
+ * foreign stage receipt remains pending, as does a recovery guard held by a live or unknown recoverer. The guard names
347
+ * the recovering process, so a recovery that itself died is taken over on exact proof of that death. */
348
+ async function recoveryGuard(guard) {
349
+ try {
350
+ return await acquireIdentityLock(guard);
351
+ }
352
+ catch (error) {
353
+ const code = codeOf(error);
354
+ if (code === 'owner-busy')
355
+ return pending();
356
+ if (code === 'unavailable')
357
+ throw new SpongeError('unavailable');
358
+ throw error;
359
+ }
360
+ }
273
361
  export async function recoverModelPackage(cacheRoot, manifest) {
274
- const plan = modelSetupPlan(manifest), root = await privateRoot(cacheRoot), lock = join(root, `.${plan.fingerprint}.lock`);
362
+ const plan = modelSetupPlan(manifest), root = await privateRoot(cacheRoot), lock = join(root, `.${plan.fingerprint}.lock`), guard = join(root, `.${plan.fingerprint}.recovery-lock`);
275
363
  let owner;
364
+ // A recovery that died after unlinking the owner lock leaves only its guard: clear it when its holder is provably dead.
276
365
  try {
277
366
  owner = await readOwner(lock, plan.fingerprint);
278
367
  }
279
368
  catch (error) {
280
- if (error.code === 'ENOENT')
369
+ if (error.code === 'ENOENT') {
370
+ if (await exists(guard))
371
+ await (await recoveryGuard(guard)).release();
281
372
  return { recovered: false };
373
+ }
282
374
  return pending();
283
375
  }
284
376
  try {
285
377
  if (await identityAlive(owner.identity))
286
- throw new ModelSetupError('owner-busy');
378
+ throw new SpongeError('owner-busy');
287
379
  }
288
380
  catch (error) {
289
- if (error instanceof ModelSetupError)
381
+ if (codeOf(error) === 'owner-busy')
290
382
  throw error;
291
383
  return pending();
292
384
  }
293
- const file = join(root, `.model-owner-${owner.incarnation}.json`), guard = join(root, `.${plan.fingerprint}.recovery-lock`);
385
+ const file = join(root, `.model-owner-${owner.incarnation}.json`);
294
386
  await sameFile(file, lock);
295
- try {
296
- await link(file, guard);
297
- await syncDirectory(root);
298
- }
299
- catch (error) {
300
- if (error.code === 'EEXIST')
301
- return pending();
302
- throw error;
303
- }
387
+ const held = await recoveryGuard(guard);
304
388
  try {
305
389
  await sameFile(file, lock);
306
390
  if (await identityAlive(owner.identity))
307
- throw new ModelSetupError('owner-busy');
391
+ throw new SpongeError('owner-busy');
308
392
  await cleanupStage(root, owner, file);
309
393
  await sameFile(file, lock);
310
394
  await unlink(lock);
311
- await syncDirectory(root);
395
+ await syncDirectoryAsync(root);
312
396
  return { recovered: true };
313
397
  }
314
398
  finally {
315
- await sameFile(file, guard);
316
- await unlink(guard);
317
- await syncDirectory(root);
399
+ await held.release();
318
400
  try {
319
401
  await lstat(lock);
320
402
  }
321
403
  catch (error) {
322
404
  if (error.code === 'ENOENT') {
323
405
  await unlink(file);
324
- await syncDirectory(root);
406
+ await syncDirectoryAsync(root);
325
407
  }
326
408
  else
327
409
  throw error;
328
410
  }
329
411
  }
330
412
  }
331
- /** Explicit setup only. Never called by install hooks or by inference fallback. */
413
+ /** Explicit setup only. Never called by install hooks or by inference fallback, except `reuseOnly`: that adopts an
414
+ * installed package from verified local artifacts under the same setup lock and never reaches the network. */
332
415
  export async function provisionModelPackage(cacheRoot, manifest, options = {}) {
333
416
  const plan = modelSetupPlan(manifest), root = await privateRoot(cacheRoot), destination = join(root, plan.fingerprint);
334
417
  if (options.signal?.aborted)
335
- throw new ModelSetupError('cancelled');
418
+ throw new SpongeError('cancelled');
336
419
  try {
337
420
  await lstat(destination);
338
421
  return await inspectModelPackage(root, manifest);
@@ -341,25 +424,36 @@ export async function provisionModelPackage(cacheRoot, manifest, options = {}) {
341
424
  if (error.code !== 'ENOENT')
342
425
  throw error;
343
426
  }
427
+ // Nothing to adopt takes no lock and writes nothing.
428
+ if (options.reuseOnly)
429
+ for (const artifact of manifest.artifacts)
430
+ if (!(await reuseCandidates(root, artifact, plan.fingerprint)).length)
431
+ throw new SpongeError('unavailable');
344
432
  const lock = join(root, `.${plan.fingerprint}.lock`), incarnation = randomUUID(), ownerFile = join(root, `.model-owner-${incarnation}.json`);
345
433
  const identity = await processIdentity(process.pid);
346
434
  if (!identity)
347
435
  pending();
348
- const owner = { schemaVersion: 1, incarnation, fingerprint: plan.fingerprint, stage: '.model-install-' + incarnation, identity: identity };
436
+ const owner = {
437
+ schemaVersion: 1,
438
+ incarnation,
439
+ fingerprint: plan.fingerprint,
440
+ stage: '.model-install-' + incarnation,
441
+ identity: identity,
442
+ };
349
443
  await writeJournal(ownerFile, owner);
350
444
  let linked = false;
351
445
  try {
352
446
  await link(ownerFile, lock);
353
447
  linked = true;
354
- await syncDirectory(root);
448
+ await syncDirectoryAsync(root);
355
449
  }
356
450
  catch (error) {
357
451
  if (linked)
358
452
  return pending();
359
453
  await unlink(ownerFile);
360
- await syncDirectory(root);
454
+ await syncDirectoryAsync(root);
361
455
  if (error.code === 'EEXIST')
362
- throw new ModelSetupError('owner-busy');
456
+ throw new SpongeError('owner-busy');
363
457
  throw error;
364
458
  }
365
459
  let stage;
@@ -371,38 +465,42 @@ export async function provisionModelPackage(cacheRoot, manifest, options = {}) {
371
465
  needed += artifact.bytes;
372
466
  const disk = await statfs(root);
373
467
  if (disk.bavail * disk.bsize < needed)
374
- throw new ModelSetupError('limit-exceeded');
468
+ throw new SpongeError('limit-exceeded');
375
469
  stage = join(root, owner.stage);
376
470
  await mkdir(stage, { mode: 0o700 });
377
471
  await chmod(stage, 0o700);
378
472
  await link(ownerFile, join(stage, 'ownership.json'));
379
- await syncDirectory(stage);
380
- await syncDirectory(root);
473
+ await syncDirectoryAsync(stage);
474
+ await syncDirectoryAsync(root);
381
475
  for (const artifact of manifest.artifacts) {
382
476
  if (await reuseArtifact(root, stage, artifact, plan.fingerprint, options.signal))
383
477
  continue;
384
- const signal = options.signal ? AbortSignal.any([options.signal, AbortSignal.timeout(600000)]) : AbortSignal.timeout(600000);
478
+ if (options.reuseOnly)
479
+ throw new SpongeError('unavailable');
480
+ const signal = options.signal
481
+ ? AbortSignal.any([options.signal, AbortSignal.timeout(600000)])
482
+ : AbortSignal.timeout(600000);
385
483
  const response = await (options.fetch ?? fetch)(artifact.url, { signal });
386
484
  if (!response.ok || !response.body || (response.url && new URL(response.url).protocol !== 'https:'))
387
- throw new ModelSetupError('unavailable');
485
+ throw new SpongeError('unavailable');
388
486
  const declared = response.headers.get('content-length');
389
487
  if (declared !== null && Number(declared) !== artifact.bytes) {
390
488
  await response.body.cancel();
391
- throw new ModelSetupError('integrity');
489
+ throw new SpongeError('integrity');
392
490
  }
393
491
  await writeStaged(join(stage, artifact.name), artifact, response.body, signal);
394
492
  }
395
493
  if (options.signal?.aborted)
396
- throw new ModelSetupError('cancelled');
397
- await syncDirectory(stage);
494
+ throw new SpongeError('cancelled');
495
+ await syncDirectoryAsync(stage);
398
496
  await rename(stage, destination);
399
497
  stage = undefined;
400
- await syncDirectory(root);
498
+ await syncDirectoryAsync(root);
401
499
  return await inspectModelPackage(root, manifest);
402
500
  }
403
501
  catch (error) {
404
502
  if (options.signal?.aborted)
405
- throw new ModelSetupError('cancelled');
503
+ throw new SpongeError('cancelled');
406
504
  throw error;
407
505
  }
408
506
  finally {
@@ -411,6 +509,39 @@ export async function provisionModelPackage(cacheRoot, manifest, options = {}) {
411
509
  await sameFile(ownerFile, lock);
412
510
  await unlink(lock);
413
511
  await unlink(ownerFile);
414
- await syncDirectory(root);
512
+ await syncDirectoryAsync(root);
513
+ }
514
+ }
515
+ const absent = (error) => error.code === 'ENOENT';
516
+ /** The installed package for this manifest, by content, or null when none is. A package installed under an earlier
517
+ * URL-bound fingerprint is adopted once: its verified artifacts are hard-linked into the content-named directory under
518
+ * the setup lock (no network; the old directory stays intact for any process holding its paths until runtime GC
519
+ * supersedes it). Anything short of a fully verified adoption is reported as not installed, as before. */
520
+ export async function inspectInstalledModelPackage(cacheRoot, manifest, options = {}) {
521
+ try {
522
+ return await inspectModelPackage(cacheRoot, manifest);
415
523
  }
524
+ catch (error) {
525
+ if (!absent(error))
526
+ throw error;
527
+ }
528
+ try {
529
+ return await provisionModelPackage(cacheRoot, manifest, { reuseOnly: true });
530
+ }
531
+ catch (error) {
532
+ if (codeOf(error) !== 'owner-busy')
533
+ return null;
534
+ }
535
+ // A concurrent adoption or setup of this package is linking or downloading: briefly wait for its verified result.
536
+ for (let i = 0; i < 20; i++) {
537
+ await (options.pause ?? ((ms) => new Promise((r) => setTimeout(r, ms))))(100);
538
+ try {
539
+ return await inspectModelPackage(cacheRoot, manifest);
540
+ }
541
+ catch (error) {
542
+ if (!absent(error))
543
+ throw error;
544
+ }
545
+ }
546
+ return null;
416
547
  }
@@ -2,7 +2,7 @@ import type { ModelPackage } from './artifacts.js';
2
2
  export type NativeKind = 'asr' | 'embedding';
3
3
  export type NativeRole = 'asr-helper' | 'embedding-helper' | 'embedding-library' | 'notices';
4
4
  export declare const nativeAssetNames: Record<NativeRole, string>;
5
- export interface NativeArtifact {
5
+ interface NativeArtifact {
6
6
  role: NativeRole;
7
7
  name: string;
8
8
  bytes: number;
@@ -23,13 +23,10 @@ export interface NativeCatalog {
23
23
  clangVersion: string;
24
24
  };
25
25
  }
26
- export declare class NativeCatalogError extends Error {
27
- readonly code: 'runtime-missing' | 'unsupported' | 'integrity';
28
- constructor(code: 'runtime-missing' | 'unsupported' | 'integrity');
29
- }
30
26
  /** Package-embedded release hashes are authority; local probe helper hashes never enter this catalog. */
31
27
  export declare function parseNativeCatalog(value: unknown, expectedVersion: string): NativeCatalog;
32
28
  export declare function nativePlatformSupported(platform?: NodeJS.Platform, architecture?: NodeJS.Architecture, kernel?: string): boolean;
33
29
  export declare function nativeCatalogPath(): string;
34
30
  export declare function loadNativeCatalog(expectedVersion: string, path?: string): Promise<NativeCatalog>;
35
31
  export declare function nativeModelPackage(catalog: NativeCatalog, kind: NativeKind): ModelPackage;
32
+ export {};