@hraness/sponge 0.2.6 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +21 -9
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +38 -16
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +9 -46
  64. package/dist/src/digest/editorial.js +57 -49
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +49 -28
  67. package/dist/src/digest/pipeline.js +343 -100
  68. package/dist/src/digest/schema.d.ts +11 -25
  69. package/dist/src/digest/schema.js +86 -18
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +3 -6
  82. package/dist/src/importers/jungle-digest.js +196 -69
  83. package/dist/src/importers/jungle-store.d.ts +15 -6
  84. package/dist/src/importers/jungle-store.js +122 -28
  85. package/dist/src/importers/jungle.d.ts +14 -14
  86. package/dist/src/importers/jungle.js +136 -31
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +106 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -79
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -0,0 +1,196 @@
1
+ import { constants } from 'node:fs';
2
+ import { open, link, lstat, rename, unlink, readdir, rm } from 'node:fs/promises';
3
+ import { basename, dirname, join } from 'node:path';
4
+ import { randomUUID } from 'node:crypto';
5
+ import { readJournal } from './journal.js';
6
+ import { syncDirectoryAsync } from '../util/fsync.js';
7
+ import { processIdentity, identityAlive } from './identity.js';
8
+ import { SpongeError } from '../errors.js';
9
+ const code = (error) => error.code;
10
+ const busy = () => new SpongeError('owner-busy');
11
+ const ownerName = (lock, pid, incarnation) => `.${basename(lock)}.${pid}.${incarnation}.owner`;
12
+ const ownerPattern = (lock) => new RegExp(`^\\.${basename(lock).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.(\\d{1,10})\\.([0-9a-f-]{36})\\.owner$`);
13
+ function record(value) {
14
+ return value?.schemaVersion === 1 &&
15
+ value.kind === 'identity-lock' &&
16
+ typeof value.incarnation === 'string' &&
17
+ /^[0-9a-f-]{36}$/.test(value.incarnation) &&
18
+ Number.isSafeInteger(value.identity?.pid) &&
19
+ value.identity.pid > 0 &&
20
+ value.identity.uid === process.getuid?.() &&
21
+ typeof value.identity.started === 'string' &&
22
+ value.identity.started.trim()
23
+ ? value
24
+ : null;
25
+ }
26
+ /** Exact death proof, or false: an unavailable process table means unknown, never dead. */
27
+ async function provablyDead(identity) {
28
+ try {
29
+ return !(await identityAlive(identity));
30
+ }
31
+ catch {
32
+ return false;
33
+ }
34
+ }
35
+ /** The recorded holder of `path`, when it is a well-formed identity lock; null for legacy, foreign or unreadable locks. */
36
+ export async function lockHolder(path) {
37
+ try {
38
+ return record(await readJournal(path));
39
+ }
40
+ catch {
41
+ return null;
42
+ }
43
+ }
44
+ /** Remove one stale lock, compare-and-swap style: rename it aside and keep it removed only if it is the exact inode
45
+ * judged dead. Anything else is relinked (if the slot is still free) and reported busy. */
46
+ async function takeOver(path) {
47
+ const dir = dirname(path);
48
+ let before;
49
+ try {
50
+ before = await lstat(path);
51
+ }
52
+ catch (error) {
53
+ if (code(error) === 'ENOENT')
54
+ return true;
55
+ throw error;
56
+ }
57
+ if (!before.isFile() || before.isSymbolicLink())
58
+ return false;
59
+ const holder = await lockHolder(path);
60
+ if (!holder)
61
+ return false;
62
+ try {
63
+ const again = await lstat(path);
64
+ if (again.ino !== before.ino || again.dev !== before.dev)
65
+ return false;
66
+ }
67
+ catch (error) {
68
+ if (code(error) === 'ENOENT')
69
+ return true;
70
+ throw error;
71
+ }
72
+ if (!(await provablyDead(holder.identity)))
73
+ return false;
74
+ const aside = join(dir, `.${basename(path)}.stale-${randomUUID()}`);
75
+ try {
76
+ await rename(path, aside);
77
+ }
78
+ catch (error) {
79
+ if (code(error) === 'ENOENT')
80
+ return true;
81
+ throw error;
82
+ }
83
+ const moved = await lstat(aside);
84
+ if (moved.ino !== before.ino || moved.dev !== before.dev) {
85
+ try {
86
+ await link(aside, path);
87
+ }
88
+ catch {
89
+ /* the slot was taken again; its holder keeps it */
90
+ }
91
+ await unlink(aside);
92
+ await syncDirectoryAsync(dir);
93
+ return false;
94
+ }
95
+ await unlink(aside);
96
+ const owner = join(dir, ownerName(path, holder.identity.pid, holder.incarnation));
97
+ try {
98
+ const s = await lstat(owner);
99
+ if (s.isFile() && s.ino === before.ino && s.dev === before.dev)
100
+ await unlink(owner);
101
+ }
102
+ catch (error) {
103
+ if (code(error) !== 'ENOENT')
104
+ throw error;
105
+ }
106
+ await syncDirectoryAsync(dir);
107
+ return true;
108
+ }
109
+ /** Owner files that never became (or no longer are) a lock, written by a process that is provably dead. */
110
+ async function sweepOwners(path, keep) {
111
+ const dir = dirname(path), pattern = ownerPattern(path);
112
+ let removed = false;
113
+ for (const name of await readdir(dir)) {
114
+ const match = pattern.exec(name);
115
+ if (!match || name === keep)
116
+ continue;
117
+ const file = join(dir, name);
118
+ try {
119
+ const s = await lstat(file);
120
+ if (!s.isFile() || s.nlink !== 1 || s.uid !== process.getuid?.())
121
+ continue;
122
+ const holder = await lockHolder(file);
123
+ // A torn write has no readable identity: only a pid with no process at all proves its writer is gone.
124
+ if (holder
125
+ ? await provablyDead(holder.identity)
126
+ : await processIdentity(Number(match[1])).then((x) => x === null, () => false)) {
127
+ const again = await lstat(file);
128
+ if (again.nlink === 1 && again.ino === s.ino) {
129
+ await unlink(file);
130
+ removed = true;
131
+ }
132
+ }
133
+ }
134
+ catch (error) {
135
+ if (code(error) !== 'ENOENT')
136
+ throw error;
137
+ }
138
+ }
139
+ if (removed)
140
+ await syncDirectoryAsync(dir);
141
+ }
142
+ export async function acquireIdentityLock(path) {
143
+ const dir = dirname(path), identity = await processIdentity(process.pid);
144
+ if (!identity)
145
+ throw new SpongeError('unavailable');
146
+ const incarnation = randomUUID(), file = join(dir, ownerName(path, identity.pid, incarnation));
147
+ const handle = await open(file, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
148
+ try {
149
+ try {
150
+ await handle.writeFile(JSON.stringify({ schemaVersion: 1, kind: 'identity-lock', incarnation, identity }));
151
+ await handle.sync();
152
+ }
153
+ finally {
154
+ await handle.close();
155
+ }
156
+ for (let attempt = 0;; attempt++) {
157
+ try {
158
+ await link(file, path);
159
+ await syncDirectoryAsync(dir);
160
+ break;
161
+ }
162
+ catch (error) {
163
+ if (code(error) !== 'EEXIST')
164
+ throw error;
165
+ }
166
+ if (attempt > 1 || !(await takeOver(path)))
167
+ throw busy();
168
+ }
169
+ }
170
+ catch (error) {
171
+ await rm(file, { force: true });
172
+ await syncDirectoryAsync(dir);
173
+ throw error;
174
+ }
175
+ await sweepOwners(path, basename(file));
176
+ return {
177
+ path,
178
+ release: async () => {
179
+ let held = false;
180
+ try {
181
+ const [a, b] = await Promise.all([lstat(file), lstat(path)]);
182
+ held = a.ino === b.ino && a.dev === b.dev;
183
+ }
184
+ catch (error) {
185
+ if (code(error) !== 'ENOENT')
186
+ throw error;
187
+ }
188
+ if (held)
189
+ await unlink(path);
190
+ await unlink(file);
191
+ await syncDirectoryAsync(dir);
192
+ if (!held)
193
+ throw new SpongeError('integrity');
194
+ },
195
+ };
196
+ }
@@ -1,11 +1,11 @@
1
1
  import { type LibraryStore } from '../store/index.js';
2
- export type RuntimeKind = 'capture' | 'model' | 'worker';
3
- export interface RuntimeEntry {
2
+ type RuntimeKind = 'capture' | 'model' | 'worker';
3
+ interface RuntimeEntry {
4
4
  kind: RuntimeKind;
5
5
  name: string;
6
6
  bytes: number;
7
7
  }
8
- export interface RuntimeKept {
8
+ interface RuntimeKept {
9
9
  kind: RuntimeKind;
10
10
  name: string;
11
11
  reason: 'referenced' | 'in-use' | 'symlink' | 'foreign' | 'catalog-unavailable' | 'current-missing';
@@ -28,3 +28,4 @@ export interface RuntimeGcOptions {
28
28
  * Applying holds worker custody, so no worker, capture, import verification, index run or query
29
29
  * can start or be running; each model directory is additionally taken under its setup lock. */
30
30
  export declare function collectRuntimes(store: LibraryStore, options: RuntimeGcOptions): Promise<RuntimeGcReport>;
31
+ export {};
@@ -1,19 +1,25 @@
1
1
  import { constants } from 'node:fs';
2
2
  import { lstat, readdir, realpath, rename, rm, link, unlink, open } from 'node:fs/promises';
3
3
  import { join } from 'node:path';
4
- import { createHash, randomUUID } from 'node:crypto';
5
- import { StoreError } from '../store/index.js';
4
+ import { randomUUID } from 'node:crypto';
5
+ import { sha256 } from '../util/hash.js';
6
6
  import { modelSetupPlan } from '../models/artifacts.js';
7
7
  import { acquireWorkerCustody } from './custody.js';
8
- import { readJournal, writeJournal, syncDir } from './journal.js';
8
+ import { readJournal, writeJournal } from './journal.js';
9
+ import { syncDirectoryAsync } from '../util/fsync.js';
9
10
  import { processIdentity, identityAlive } from './identity.js';
11
+ import { SpongeError } from '../errors.js';
10
12
  const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
11
- const patterns = { capture: /^capture-[a-z0-9][a-z0-9.-]{0,99}$/, model: /^[a-f0-9]{64}$/, worker: uuid };
13
+ const patterns = {
14
+ capture: /^capture-[a-z0-9][a-z0-9.-]{0,99}$/,
15
+ model: /^[a-f0-9]{64}$/,
16
+ worker: uuid,
17
+ };
12
18
  const helperNames = ['sponge-speech', 'sponge-embedding'];
13
19
  const trash = /^\.gc-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
14
20
  // A worker writes owner.json just after creating its directory; a young unreadable directory may be mid-acquisition.
15
21
  const youngMs = 10 * 60 * 1000;
16
- const integrity = () => new StoreError('integrity');
22
+ const integrity = () => new SpongeError('integrity');
17
23
  const missing = (error) => error.code === 'ENOENT';
18
24
  /** Disk blocks removing one tree frees, never following links: each inode once, and a file only when all of its
19
25
  * hard links are inside the tree (a model artifact shared with a kept package frees nothing). */
@@ -59,19 +65,25 @@ async function parent(path) {
59
65
  return null;
60
66
  throw error;
61
67
  }
62
- if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid?.() || (stat.mode & 0o077) !== 0 || await realpath(path) !== path)
68
+ if (!stat.isDirectory() ||
69
+ stat.isSymbolicLink() ||
70
+ stat.uid !== process.getuid?.() ||
71
+ (stat.mode & 0o077) !== 0 ||
72
+ (await realpath(path)) !== path)
63
73
  throw integrity();
64
74
  return path;
65
75
  }
66
- async function exists(path) { try {
67
- await lstat(path);
68
- return true;
76
+ async function exists(path) {
77
+ try {
78
+ await lstat(path);
79
+ return true;
80
+ }
81
+ catch (error) {
82
+ if (missing(error))
83
+ return false;
84
+ throw error;
85
+ }
69
86
  }
70
- catch (error) {
71
- if (missing(error))
72
- return false;
73
- throw error;
74
- } }
75
87
  async function fileSha256(path, maxBytes) {
76
88
  let file;
77
89
  try {
@@ -84,7 +96,7 @@ async function fileSha256(path, maxBytes) {
84
96
  const stat = await file.stat();
85
97
  if (!stat.isFile() || stat.size > maxBytes)
86
98
  return null;
87
- return createHash('sha256').update(await file.readFile()).digest('hex');
99
+ return sha256(await file.readFile());
88
100
  }
89
101
  finally {
90
102
  await file.close();
@@ -123,9 +135,9 @@ async function remove(candidate) {
123
135
  await rename(moved, path);
124
136
  return false;
125
137
  }
126
- await syncDir(candidate.dir);
138
+ await syncDirectoryAsync(candidate.dir);
127
139
  await rm(moved, { recursive: true });
128
- await syncDir(candidate.dir);
140
+ await syncDirectoryAsync(candidate.dir);
129
141
  return true;
130
142
  }
131
143
  async function workerInUse(dir, name) {
@@ -151,26 +163,30 @@ async function workerInUse(dir, name) {
151
163
  * can start or be running; each model directory is additionally taken under its setup lock. */
152
164
  export async function collectRuntimes(store, options) {
153
165
  const dryRun = options.dryRun === true, base = await realpath(store.root), runtime = await parent(join(base, 'runtime'));
154
- if (store.ownerSnapshot())
155
- throw new StoreError('owner-busy');
156
- if (store.pendingCleanupCount())
157
- throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
166
+ if (store.queue.ownerSnapshot())
167
+ throw new SpongeError('owner-busy');
168
+ if (store.queue.pendingCleanup())
169
+ throw new SpongeError('cleanup-required');
158
170
  const lockPath = join(base, 'runtime', 'worker.lock');
159
171
  // Checked first so a refused collection leaves no custody journal; acquisition stays the authority.
160
172
  if (await exists(lockPath))
161
- throw new StoreError('owner-busy');
173
+ throw new SpongeError('owner-busy');
162
174
  const custody = dryRun || !runtime ? undefined : await acquireWorkerCustody(base);
163
175
  const removed = [], kept = [];
164
176
  try {
165
177
  if (!runtime)
166
178
  return { dryRun, removed, kept, bytes: 0 };
167
179
  // Recheck under custody: an owner that appeared before our lock still blocks.
168
- if (store.ownerSnapshot())
169
- throw new StoreError('owner-busy');
170
- if (store.pendingCleanupCount())
171
- throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
172
- const dirs = { capture: await parent(join(runtime, 'capture')), model: await parent(join(runtime, 'models')), worker: await parent(join(runtime, 'workers')) };
173
- const references = store.runtimeReferences(), models = options.models;
180
+ if (store.queue.ownerSnapshot())
181
+ throw new SpongeError('owner-busy');
182
+ if (store.queue.pendingCleanup())
183
+ throw new SpongeError('cleanup-required');
184
+ const dirs = {
185
+ capture: await parent(join(runtime, 'capture')),
186
+ model: await parent(join(runtime, 'models')),
187
+ worker: await parent(join(runtime, 'workers')),
188
+ };
189
+ const references = store.blobs.runtimeReferences(), models = options.models;
174
190
  const modelKeep = new Set(models?.fingerprints ?? []);
175
191
  for (const artifact of references.modelArtifacts) {
176
192
  if (patterns.model.test(artifact.id))
@@ -178,22 +194,24 @@ export async function collectRuntimes(store, options) {
178
194
  try {
179
195
  modelKeep.add(modelSetupPlan(JSON.parse(artifact.manifest)).fingerprint);
180
196
  }
181
- catch { /* An unparsable row still pins its id. */ }
197
+ catch {
198
+ /* An unparsable row still pins its id. */
199
+ }
182
200
  }
183
201
  // A generation's helper is satisfied only by an installed kept package, never by an uninstalled catalog entry.
184
202
  const helpers = new Set(references.embeddingHelpers), installed = new Set();
185
203
  for (const fp of modelKeep) {
186
- if (!dirs.model || !await exists(join(dirs.model, fp)))
204
+ if (!dirs.model || !(await exists(join(dirs.model, fp))))
187
205
  continue;
188
206
  for (const helper of helperNames) {
189
207
  if (await exists(join(dirs.model, fp, helper)))
190
208
  installed.add(helper);
191
209
  }
192
- helpers.delete(await fileSha256(join(dirs.model, fp, 'sponge-embedding'), 32 * 1024 ** 2) ?? '');
210
+ helpers.delete((await fileSha256(join(dirs.model, fp, 'sponge-embedding'), 32 * 1024 ** 2)) ?? '');
193
211
  }
194
212
  const take = async (candidate) => {
195
213
  const bytes = await diskBytes(join(candidate.dir, candidate.name));
196
- if (dryRun || await remove(candidate))
214
+ if (dryRun || (await remove(candidate)))
197
215
  removed.push({ kind: candidate.kind, name: candidate.name, bytes });
198
216
  else
199
217
  kept.push({ kind: candidate.kind, name: candidate.name, reason: 'symlink' });
@@ -219,21 +237,22 @@ export async function collectRuntimes(store, options) {
219
237
  continue;
220
238
  }
221
239
  const helper = join(dir, name, 'sponge-embedding');
222
- if ((references.unmappedSpaces && await exists(helper)) || helpers.size && helpers.has(await fileSha256(helper, 32 * 1024 ** 2) ?? '')) {
240
+ if ((references.unmappedSpaces && (await exists(helper))) ||
241
+ (helpers.size && helpers.has((await fileSha256(helper, 32 * 1024 ** 2)) ?? ''))) {
223
242
  kept.push({ kind: 'model', name, reason: 'referenced' });
224
243
  continue;
225
244
  }
226
245
  // Until this Sponge's package of a kind is installed, older packages of that kind are its only local copy.
227
246
  let missingKind = false;
228
247
  for (const helper of helperNames)
229
- if (!installed.has(helper) && await exists(join(dir, name, helper)))
248
+ if (!installed.has(helper) && (await exists(join(dir, name, helper))))
230
249
  missingKind = true;
231
250
  if (missingKind) {
232
251
  kept.push({ kind: 'model', name, reason: 'current-missing' });
233
252
  continue;
234
253
  }
235
254
  const lock = join(dir, `.${name}.lock`);
236
- if (await exists(lock) || await exists(join(dir, `.${name}.recovery-lock`))) {
255
+ if ((await exists(lock)) || (await exists(join(dir, `.${name}.recovery-lock`)))) {
237
256
  kept.push({ kind: 'model', name, reason: 'in-use' });
238
257
  continue;
239
258
  }
@@ -246,7 +265,13 @@ export async function collectRuntimes(store, options) {
246
265
  if (!identity)
247
266
  throw integrity();
248
267
  const incarnation = randomUUID(), ownerFile = join(dir, `.model-owner-${incarnation}.json`);
249
- await writeJournal(ownerFile, { schemaVersion: 1, incarnation, fingerprint: name, stage: '.model-install-' + incarnation, identity });
268
+ await writeJournal(ownerFile, {
269
+ schemaVersion: 1,
270
+ incarnation,
271
+ fingerprint: name,
272
+ stage: '.model-install-' + incarnation,
273
+ identity,
274
+ });
250
275
  try {
251
276
  try {
252
277
  await link(ownerFile, lock);
@@ -259,7 +284,7 @@ export async function collectRuntimes(store, options) {
259
284
  throw error;
260
285
  }
261
286
  try {
262
- await syncDir(dir);
287
+ await syncDirectoryAsync(dir);
263
288
  await take(candidate);
264
289
  }
265
290
  finally {
@@ -268,14 +293,14 @@ export async function collectRuntimes(store, options) {
268
293
  }
269
294
  finally {
270
295
  await unlink(ownerFile);
271
- await syncDir(dir);
296
+ await syncDirectoryAsync(dir);
272
297
  }
273
298
  }
274
299
  for (const candidate of await children('worker', dirs.worker, kept)) {
275
300
  const { name, dir } = candidate;
276
301
  if (name === custody?.incarnation)
277
302
  continue;
278
- if (!trash.test(name) && await workerInUse(dir, name)) {
303
+ if (!trash.test(name) && (await workerInUse(dir, name))) {
279
304
  kept.push({ kind: 'worker', name, reason: 'in-use' });
280
305
  continue;
281
306
  }
@@ -288,9 +313,11 @@ export async function collectRuntimes(store, options) {
288
313
  await custody.release();
289
314
  try {
290
315
  await rm(custody.directory, { recursive: true });
291
- await syncDir(join(base, 'runtime', 'workers'));
316
+ await syncDirectoryAsync(join(base, 'runtime', 'workers'));
317
+ }
318
+ catch {
319
+ /* The next collection removes this settled journal. */
292
320
  }
293
- catch { /* The next collection removes this settled journal. */ }
294
321
  }
295
322
  }
296
323
  }
@@ -13,9 +13,14 @@ const abort = () => controller.abort();
13
13
  process.on('disconnect', abort);
14
14
  process.on('SIGTERM', abort);
15
15
  process.on('SIGINT', abort);
16
- await writeJournal(join(root, ticketId + '.supervisor.json'), { ...pending, identity: await processIdentity(process.pid) });
17
- const deadline = setTimeout(() => { if (!started)
18
- process.exit(2); }, 10000);
16
+ await writeJournal(join(root, ticketId + '.supervisor.json'), {
17
+ ...pending,
18
+ identity: await processIdentity(process.pid),
19
+ });
20
+ const deadline = setTimeout(() => {
21
+ if (!started)
22
+ process.exit(2);
23
+ }, 10000);
19
24
  process.once('message', async (message) => {
20
25
  if (message?.type !== 'run' || message.ticket !== pending.ticket || message.attemptId !== pending.attemptId)
21
26
  process.exit(2);
@@ -23,11 +28,19 @@ process.once('message', async (message) => {
23
28
  clearTimeout(deadline);
24
29
  const result = await runOwned(message.executable, message.args, { ...message.options, signal: controller.signal });
25
30
  if (result.cleanup === 'settled')
26
- await writeJournal(join(root, ticketId + '.settled.json'), { schemaVersion: 1, ticket: pending.ticket, attemptId: pending.attemptId, ownerIncarnation: pending.ownerIncarnation, cleanup: 'settled' });
31
+ await writeJournal(join(root, ticketId + '.settled.json'), {
32
+ schemaVersion: 1,
33
+ ticket: pending.ticket,
34
+ attemptId: pending.attemptId,
35
+ ownerIncarnation: pending.ownerIncarnation,
36
+ cleanup: 'settled',
37
+ });
27
38
  if (process.connected)
28
39
  process.send?.({ type: 'result', result }, undefined, undefined, () => process.disconnect());
29
40
  });
30
- process.on('message', (message) => { if (message?.type === 'cancel')
31
- abort(); });
41
+ process.on('message', (message) => {
42
+ if (message?.type === 'cancel')
43
+ abort();
44
+ });
32
45
  if (process.connected)
33
46
  process.send({ type: 'ready', ticket: pending.ticket });
@@ -12,19 +12,41 @@ export function supervisedRunner(ownerDirectory, ownerIncarnation, attemptId) {
12
12
  const root = await directory(join(ownerDirectory, 'attempts', attemptId)), ticket = randomUUID();
13
13
  await writeJournal(join(root, ticket + '.pending.json'), { schemaVersion: 1, ticket, attemptId, ownerIncarnation });
14
14
  const source = import.meta.url.endsWith('.ts'), entry = fileURLToPath(new URL(source ? './supervisor-child.ts' : './supervisor-child.js', import.meta.url));
15
- return new Promise(resolve => {
16
- const child = fork(entry, [root, ticket], { detached: true, stdio: ['ignore', 'ignore', 'ignore', 'ipc'], execArgv: source ? ['--import', import.meta.resolve('tsx')] : [], env: { PATH: '/usr/bin:/bin', LANG: 'en_US.UTF-8' } });
15
+ return new Promise((resolve) => {
16
+ const child = fork(entry, [root, ticket], {
17
+ detached: true,
18
+ stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
19
+ execArgv: source ? ['--import', import.meta.resolve('tsx')] : [],
20
+ env: { PATH: '/usr/bin:/bin', LANG: 'en_US.UTF-8' },
21
+ });
17
22
  let result, finished = false, dispatched = false;
18
- const cancel = () => { if (dispatched && child.connected)
19
- child.send({ type: 'cancel' }, () => { }); };
23
+ const cancel = () => {
24
+ if (dispatched && child.connected)
25
+ child.send({ type: 'cancel' }, () => { });
26
+ };
20
27
  options.signal?.addEventListener('abort', cancel, { once: true });
21
28
  child.on('message', (message) => {
22
29
  if (message?.type === 'result' && dispatched)
23
30
  result = message.result;
24
31
  if (message?.type === 'ready' && message.ticket === ticket && !dispatched) {
25
32
  dispatched = true;
26
- child.send({ type: 'run', ticket, attemptId, executable, args, options: { cwd: options.cwd, env: options.env, deadlineMs: options.deadlineMs, maxOutputBytes: options.maxOutputBytes, graceMs: options.graceMs } }, () => { if (options.signal?.aborted)
27
- cancel(); });
33
+ child.send({
34
+ type: 'run',
35
+ ticket,
36
+ attemptId,
37
+ executable,
38
+ args,
39
+ options: {
40
+ cwd: options.cwd,
41
+ env: options.env,
42
+ deadlineMs: options.deadlineMs,
43
+ maxOutputBytes: options.maxOutputBytes,
44
+ graceMs: options.graceMs,
45
+ },
46
+ }, () => {
47
+ if (options.signal?.aborted)
48
+ cancel();
49
+ });
28
50
  }
29
51
  });
30
52
  const finish = async () => {
@@ -35,10 +57,21 @@ export function supervisedRunner(ownerDirectory, ownerIncarnation, attemptId) {
35
57
  let settled = false;
36
58
  try {
37
59
  const receipt = await readJournal(join(root, ticket + '.settled.json'));
38
- settled = receipt.ticket === ticket && receipt.attemptId === attemptId && receipt.ownerIncarnation === ownerIncarnation && receipt.cleanup === 'settled';
60
+ settled =
61
+ receipt.ticket === ticket &&
62
+ receipt.attemptId === attemptId &&
63
+ receipt.ownerIncarnation === ownerIncarnation &&
64
+ receipt.cleanup === 'settled';
39
65
  }
40
66
  catch { }
41
- resolve(result && settled ? result : { code: null, stdout: '', stopped: options.signal?.aborted ? 'cancelled' : 'unavailable', cleanup: 'pending' });
67
+ resolve(result && settled
68
+ ? result
69
+ : {
70
+ code: null,
71
+ stdout: '',
72
+ stopped: options.signal?.aborted ? 'cancelled' : 'unavailable',
73
+ cleanup: 'pending',
74
+ });
42
75
  };
43
76
  child.once('error', () => void finish());
44
77
  child.once('exit', () => void finish());
@@ -27,14 +27,14 @@
27
27
  }
28
28
  ],
29
29
  "build": {
30
- "asrSourceSha256": "e944e4e0b1c2cf571e78a83fcfbf56a91a14fd08dcc701562d32555cf090ec40",
30
+ "asrSourceSha256": "2862d5869808e2f048767f075ddb234aa7535bc7242550365efcfb718cc7f7b8",
31
31
  "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
32
- "embeddingSourceSha256": "7d58718911725b3c3dce35a09a253704b4e5e732cf1e23c12389485127534b10",
32
+ "embeddingSourceSha256": "a584f43c8af9743612984804f3fa2df0d204f13481c3a2d6d06b521e7661bc2e",
33
33
  "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
34
34
  },
35
35
  "minimumMacOS": "26.0",
36
- "packageVersion": "0.2.6",
36
+ "packageVersion": "0.3.0",
37
37
  "platform": "darwin",
38
38
  "schemaVersion": 1,
39
- "sourceCommit": "1190f23ad793985b6934583c5d8b0957d79a4ce0"
39
+ "sourceCommit": "8b1dc263364669190b4bec2ed5d7141a838e0841"
40
40
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hraness/sponge",
3
- "version": "0.2.6",
3
+ "version": "0.3.0",
4
4
  "description": "A local library for captured articles and verified offline assets.",
5
5
  "repository": {
6
6
  "type": "git",
@@ -30,6 +30,8 @@
30
30
  "check": "node --check bin/sponge.js",
31
31
  "build": "tsc -p tsconfig.build.json",
32
32
  "typecheck": "tsc --noEmit",
33
+ "cycles": "madge --circular --extensions ts src",
34
+ "deadcode": "knip --no-progress",
33
35
  "test": "node --import tsx --test --test-isolation=none test/*/*.test.ts",
34
36
  "probe": "node --import tsx scripts/probes/cli.ts",
35
37
  "prepack": "npm run build",
@@ -38,11 +40,17 @@
38
40
  "verify:pdf": "node --import tsx scripts/verify-pdf.ts",
39
41
  "verify:import": "node --import tsx scripts/verify-import.ts",
40
42
  "verify:installed": "node --import tsx scripts/verify-installed.ts",
41
- "verify:installed-native": "node --import tsx scripts/verify-installed-native.ts"
43
+ "verify:installed-native": "node --import tsx scripts/verify-installed-native.ts",
44
+ "format": "prettier --write \"{bin,contracts,migrations,scripts,src,test}/**/*.{ts,js,mjs}\"",
45
+ "format:check": "prettier --check \"{bin,contracts,migrations,scripts,src,test}/**/*.{ts,js,mjs}\""
42
46
  },
43
47
  "packageManager": "npm@11.19.0",
44
48
  "devDependencies": {
49
+ "@types/mdast": "4.0.4",
45
50
  "@types/node": "24.10.1",
51
+ "knip": "6.39.0",
52
+ "madge": "8.0.0",
53
+ "prettier": "3.9.10",
46
54
  "tsx": "4.20.6",
47
55
  "typescript": "5.9.3"
48
56
  },