@hraness/sponge 0.2.6 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +21 -9
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +38 -16
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +9 -46
  64. package/dist/src/digest/editorial.js +57 -49
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +49 -28
  67. package/dist/src/digest/pipeline.js +343 -100
  68. package/dist/src/digest/schema.d.ts +11 -25
  69. package/dist/src/digest/schema.js +86 -18
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +3 -6
  82. package/dist/src/importers/jungle-digest.js +196 -69
  83. package/dist/src/importers/jungle-store.d.ts +15 -6
  84. package/dist/src/importers/jungle-store.js +122 -28
  85. package/dist/src/importers/jungle.d.ts +14 -14
  86. package/dist/src/importers/jungle.js +136 -31
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +106 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -79
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -1,41 +1,95 @@
1
- import { constants, openSync, closeSync, fstatSync, fsyncSync, unlinkSync, rmSync } from 'node:fs';
1
+ import { constants, openSync, closeSync, fstatSync, fsyncSync, unlinkSync, rmSync, chmodSync, fchmodSync, } from 'node:fs';
2
2
  import { dirname, join, resolve } from 'node:path';
3
3
  import { openLibrary } from '../store/index.js';
4
- import { safeDirectory } from '../store/files.js';
5
- import { StoreError } from '../store/types.js';
4
+ import { safeDirectory, pathExists } from '../store/files.js';
5
+ import { loadPublishKey, publishKeyPath, maximumPublishKeyBytes } from '../publish/identity.js';
6
6
  import { inspectDatabase, neutralizeRestoredDatabase } from './database.js';
7
- import { bounded, exact, fileDescriptor, invalid, maximumDatabaseBytes, maximumEntries, maximumTotalBytes, openRead, readExact, transfer, writeAll, describe, newDirectory, discardOwned, syncDirectory, writePrivateExclusive } from './files.js';
7
+ import { bounded, digestString, exact, fileDescriptor, invalid, maximumDatabaseBytes, maximumEntries, maximumTotalBytes, openRead, readExact, transfer, writeAll, describe, newDirectory, discardOwned, writePrivateExclusive, } from './files.js';
8
+ import { syncDirectory } from '../util/fsync.js';
9
+ import { SpongeError } from '../errors.js';
8
10
  const magic = Buffer.from('SPONGE-BACKUP-1\n');
9
11
  const maxManifestBytes = 32 * 1024 ** 2;
12
+ function publishKeyEntry(value) {
13
+ if (value === null)
14
+ return null;
15
+ const r = exact(value, ['path', 'bytes', 'sha256']);
16
+ if (r.path !== publishKeyPath)
17
+ invalid();
18
+ return { path: publishKeyPath, bytes: bounded(r.bytes, maximumPublishKeyBytes), sha256: digestString(r.sha256) };
19
+ }
10
20
  function manifest(value) {
11
- const r = exact(value, ['format', 'version', 'schemaVersion', 'libraryId', 'revisionEpoch', 'files']);
12
- if (r.format !== 'sponge-backup' || r.version !== 1 || typeof r.libraryId !== 'string' || r.libraryId.length > 256 || !Array.isArray(r.files) || r.files.length < 1 || r.files.length > maximumEntries + 1)
21
+ const v = value && typeof value === 'object' ? value.version : undefined;
22
+ const r = exact(value, [
23
+ 'format',
24
+ 'version',
25
+ 'schemaVersion',
26
+ 'libraryId',
27
+ 'revisionEpoch',
28
+ 'files',
29
+ ...(v === 2 ? ['publishKey'] : []),
30
+ ]);
31
+ if (r.format !== 'sponge-backup' ||
32
+ (r.version !== 1 && r.version !== 2) ||
33
+ typeof r.libraryId !== 'string' ||
34
+ r.libraryId.length > 256 ||
35
+ !Array.isArray(r.files) ||
36
+ r.files.length < 1 ||
37
+ r.files.length > maximumEntries + 1)
13
38
  invalid();
14
39
  const files = r.files.map(fileDescriptor);
15
- if (files[0]?.path !== 'library.sqlite' || files.slice(1).some(f => f.path === 'library.sqlite') || new Set(files.map(f => f.path)).size !== files.length)
40
+ if (files[0]?.path !== 'library.sqlite' ||
41
+ files.slice(1).some((f) => f.path === 'library.sqlite') ||
42
+ new Set(files.map((f) => f.path)).size !== files.length)
16
43
  invalid();
17
- return { format: 'sponge-backup', version: 1, schemaVersion: bounded(r.schemaVersion, Number.MAX_SAFE_INTEGER), libraryId: r.libraryId, revisionEpoch: bounded(r.revisionEpoch, Number.MAX_SAFE_INTEGER), files };
44
+ return {
45
+ format: 'sponge-backup',
46
+ version: r.version,
47
+ schemaVersion: bounded(r.schemaVersion, Number.MAX_SAFE_INTEGER),
48
+ libraryId: r.libraryId,
49
+ revisionEpoch: bounded(r.revisionEpoch, Number.MAX_SAFE_INTEGER),
50
+ files,
51
+ publishKey: r.version === 2 ? publishKeyEntry(r.publishKey) : null,
52
+ };
18
53
  }
19
54
  function sameSnapshot(info, m) {
20
- if (info.schemaVersion !== m.schemaVersion || info.libraryId !== m.libraryId || info.revisionEpoch !== m.revisionEpoch)
55
+ if (info.schemaVersion !== m.schemaVersion ||
56
+ info.libraryId !== m.libraryId ||
57
+ info.revisionEpoch !== m.revisionEpoch)
21
58
  invalid();
22
- const expected = info.blobs.map(b => ({ path: b.relativePath, bytes: b.bytes, sha256: b.sha256 }));
59
+ const expected = info.blobs.map((b) => ({ path: b.relativePath, bytes: b.bytes, sha256: b.sha256 }));
23
60
  if (JSON.stringify(expected) !== JSON.stringify(m.files.slice(1)))
24
61
  invalid();
25
62
  }
26
63
  export async function backupLibrary(store, output) {
27
64
  if (resolve(output) !== output)
28
- throw new StoreError('invalid-input');
65
+ throw new SpongeError('invalid-input');
29
66
  safeDirectory(dirname(output));
30
- const snapshot = await store.createBackupSnapshot();
67
+ const snapshot = await store.backups.snapshot();
31
68
  let fd, created = false;
32
69
  try {
33
70
  const info = inspectDatabase(snapshot.path);
34
- const files = [describe(snapshot.path, 'library.sqlite', maximumDatabaseBytes), ...info.blobs.map(b => ({ path: b.relativePath, bytes: b.bytes, sha256: b.sha256 }))];
35
- const m = { format: 'sponge-backup', version: 1, schemaVersion: info.schemaVersion, libraryId: info.libraryId, revisionEpoch: info.revisionEpoch, files };
71
+ const files = [
72
+ describe(snapshot.path, 'library.sqlite', maximumDatabaseBytes),
73
+ ...info.blobs.map((b) => ({ path: b.relativePath, bytes: b.bytes, sha256: b.sha256 })),
74
+ ];
75
+ // The publishing key is immutable once linked into place; a key that would not load is refused here, not discovered at restore.
76
+ const keyPath = join(store.root, publishKeyPath), publishKey = pathExists(keyPath)
77
+ ? (loadPublishKey(keyPath), describe(keyPath, publishKeyPath, maximumPublishKeyBytes))
78
+ : null;
79
+ const m = {
80
+ format: 'sponge-backup',
81
+ version: 2,
82
+ schemaVersion: info.schemaVersion,
83
+ libraryId: info.libraryId,
84
+ revisionEpoch: info.revisionEpoch,
85
+ files,
86
+ publishKey,
87
+ };
36
88
  const json = Buffer.from(JSON.stringify(m));
37
- if (json.length > maxManifestBytes || files.reduce((n, f) => n + f.bytes, json.length + magic.length + 4) > maximumTotalBytes)
38
- throw new StoreError('limit-exceeded');
89
+ if (json.length > maxManifestBytes ||
90
+ [...files, ...(publishKey ? [publishKey] : [])].reduce((n, f) => n + f.bytes, json.length + magic.length + 4) >
91
+ maximumTotalBytes)
92
+ throw new SpongeError('limit-exceeded');
39
93
  fd = openSync(output, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
40
94
  created = true;
41
95
  writeAll(fd, magic);
@@ -43,7 +97,7 @@ export async function backupLibrary(store, output) {
43
97
  length.writeUInt32BE(json.length);
44
98
  writeAll(fd, length);
45
99
  writeAll(fd, json);
46
- for (const file of files) {
100
+ for (const file of publishKey ? [...files, publishKey] : files) {
47
101
  const input = openRead(file.path === 'library.sqlite' ? snapshot.path : join(store.root, file.path));
48
102
  try {
49
103
  if (fstatSync(input).size !== file.bytes || transfer(input, file.bytes, fd) !== file.sha256)
@@ -58,7 +112,13 @@ export async function backupLibrary(store, output) {
58
112
  fd = undefined;
59
113
  syncDirectory(dirname(output));
60
114
  const result = describe(output, 'archive', maximumTotalBytes);
61
- return { output, sha256: result.sha256, bytes: result.bytes, itemsSnapshotEpoch: info.revisionEpoch };
115
+ return {
116
+ output,
117
+ sha256: result.sha256,
118
+ bytes: result.bytes,
119
+ itemsSnapshotEpoch: info.revisionEpoch,
120
+ publishKey: publishKey !== null,
121
+ };
62
122
  }
63
123
  catch (error) {
64
124
  if (fd !== undefined) {
@@ -78,7 +138,7 @@ export async function backupLibrary(store, output) {
78
138
  rmSync(snapshot.path + side, { force: true });
79
139
  }
80
140
  finally {
81
- store.finishBackup(snapshot.id);
141
+ store.backups.finish(snapshot.id);
82
142
  }
83
143
  }
84
144
  }
@@ -101,7 +161,8 @@ export async function restoreLibrary(input, into) {
101
161
  invalid();
102
162
  }
103
163
  const m = manifest(parsed);
104
- if (m.files.reduce((n, f) => n + f.bytes, magic.length + 4 + length) !== size)
164
+ if ([...m.files, ...(m.publishKey ? [m.publishKey] : [])].reduce((n, f) => n + f.bytes, magic.length + 4 + length) !==
165
+ size)
105
166
  invalid();
106
167
  target = newDirectory(into);
107
168
  for (const file of m.files) {
@@ -118,16 +179,39 @@ export async function restoreLibrary(input, into) {
118
179
  closeSync(out);
119
180
  }
120
181
  }
182
+ // The key goes only into the new directory created above (exclusive create), so restore never replaces an existing key.
183
+ if (m.publishKey) {
184
+ const dir = safeDirectory(join(target.root, 'keys'), true);
185
+ chmodSync(dir, 0o700);
186
+ const path = join(target.root, publishKeyPath), out = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
187
+ try {
188
+ if (transfer(fd, m.publishKey.bytes, out) !== m.publishKey.sha256)
189
+ invalid();
190
+ fchmodSync(out, 0o600);
191
+ fsyncSync(out);
192
+ }
193
+ finally {
194
+ closeSync(out);
195
+ }
196
+ syncDirectory(dir);
197
+ loadPublishKey(path);
198
+ }
121
199
  sameSnapshot(inspectDatabase(join(into, 'library.sqlite')), m);
122
200
  const state = neutralizeRestoredDatabase(join(into, 'library.sqlite'));
123
201
  writePrivateExclusive(join(into, '.sponge-library'), 'sponge-library-v1\n');
124
202
  // Mark bindings as deliberately absent. This never installs runtime/auth/model files.
125
203
  safeDirectory(join(into, 'runtime'), true);
126
- writePrivateExclusive(join(into, 'runtime', 'restore.json'), JSON.stringify({ schemaVersion: 1, libraryId: m.libraryId, ownerIncarnation: state.incarnation, connections: 'unresolved', heldJobs: state.heldJobs }) + '\n');
204
+ writePrivateExclusive(join(into, 'runtime', 'restore.json'), JSON.stringify({
205
+ schemaVersion: 1,
206
+ libraryId: m.libraryId,
207
+ ownerIncarnation: state.incarnation,
208
+ connections: 'unresolved',
209
+ heldJobs: state.heldJobs,
210
+ }) + '\n');
127
211
  const store = await openLibrary(into);
128
212
  store.close();
129
213
  syncDirectory(into);
130
- return { root: into, libraryId: m.libraryId, ...state };
214
+ return { root: into, libraryId: m.libraryId, ...state, publishKey: m.publishKey !== null };
131
215
  }
132
216
  catch (error) {
133
217
  if (target)
@@ -1,46 +1,78 @@
1
1
  import { DatabaseSync } from 'node:sqlite';
2
2
  import { randomUUID } from 'node:crypto';
3
- import { libraryMigrations } from '../store/index.js';
4
- import { digest, regularFile } from '../store/files.js';
5
- import { StoreError } from '../store/types.js';
3
+ import { migrations } from '../../migrations/index.js';
4
+ import { regularFile } from '../store/files.js';
5
+ import { sha256 } from '../util/hash.js';
6
6
  import { blobPath, bounded, digestString, invalid, maximumBlobBytes, maximumEntries } from './files.js';
7
- function schema(db) { return JSON.stringify(db.prepare("SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY type,name").all()); }
7
+ import { SpongeError } from '../errors.js';
8
+ function schema(db) {
9
+ return JSON.stringify(db.prepare('SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY type,name').all());
10
+ }
8
11
  /** Compare executable schema before any restored SQL mutation, not just claimed migration hashes. */
9
12
  export function inspectDatabase(path) {
10
13
  regularFile(path);
14
+ // SQLite's own errors on a forged or corrupt file (not a database, malformed image) are an integrity failure, not a crash.
15
+ try {
16
+ return inspectOpened(path);
17
+ }
18
+ catch (error) {
19
+ if (error instanceof SpongeError)
20
+ throw error;
21
+ throw new SpongeError('integrity', { cause: error });
22
+ }
23
+ }
24
+ function inspectOpened(path) {
11
25
  const db = new DatabaseSync(path, { readOnly: true });
12
26
  const expected = new DatabaseSync(':memory:');
13
27
  try {
14
28
  db.exec('PRAGMA trusted_schema=OFF');
15
29
  const version = bounded(db.prepare('PRAGMA user_version').get()?.user_version, Number.MAX_SAFE_INTEGER);
16
- if (version > libraryMigrations.length)
17
- throw new StoreError('schema-newer');
30
+ if (version > migrations.length)
31
+ throw new SpongeError('schema-newer');
18
32
  if (version < 1)
19
33
  invalid();
20
34
  for (let i = 0; i < version; i++) {
21
- expected.exec(libraryMigrations[i]);
22
- if (db.prepare('SELECT checksum FROM schema_migrations WHERE version=?').get(i + 1)?.checksum !== digest(libraryMigrations[i]))
35
+ const { sql } = migrations[i];
36
+ expected.exec(sql);
37
+ if (db.prepare('SELECT checksum FROM schema_migrations WHERE version=?').get(i + 1)?.checksum !== sha256(sql))
23
38
  invalid();
24
39
  }
25
40
  if (schema(db) !== schema(expected))
26
41
  invalid();
27
42
  if (db.prepare('SELECT count(*) n FROM schema_migrations').get()?.n !== version)
28
43
  invalid();
29
- if (db.prepare('PRAGMA quick_check').get()?.quick_check !== 'ok' || db.prepare('PRAGMA foreign_key_check').all().length)
44
+ if (db.prepare('PRAGMA quick_check').get()?.quick_check !== 'ok' ||
45
+ db.prepare('PRAGMA foreign_key_check').all().length)
30
46
  invalid();
31
47
  const info = db.prepare('SELECT uuid,schema_version,revision_epoch FROM library_info WHERE id=1').get();
32
48
  if (!info || typeof info.uuid !== 'string' || info.uuid.length > 256 || info.schema_version !== version)
33
49
  invalid();
34
50
  const count = Number(db.prepare('SELECT count(*) n FROM assets').get()?.n);
35
51
  if (count > maximumEntries)
36
- throw new StoreError('limit-exceeded');
37
- const blobs = db.prepare('SELECT sha256,bytes,relative_path FROM assets ORDER BY sha256').all().map(row => { const sha256 = digestString(row.sha256); if (row.relative_path !== blobPath(sha256))
38
- invalid(); return { sha256, bytes: bounded(row.bytes, maximumBlobBytes), relativePath: String(row.relative_path) }; });
39
- for (const row of db.prepare('SELECT markdown,sha256,bytes FROM representations WHERE markdown IS NOT NULL').iterate()) {
40
- if (typeof row.markdown !== 'string' || digest(row.markdown) !== row.sha256 || Buffer.byteLength(row.markdown) !== row.bytes)
52
+ throw new SpongeError('limit-exceeded');
53
+ const blobs = db
54
+ .prepare('SELECT sha256,bytes,relative_path FROM assets ORDER BY sha256')
55
+ .all()
56
+ .map((row) => {
57
+ const sha256 = digestString(row.sha256);
58
+ if (row.relative_path !== blobPath(sha256))
59
+ invalid();
60
+ return { sha256, bytes: bounded(row.bytes, maximumBlobBytes), relativePath: String(row.relative_path) };
61
+ });
62
+ for (const row of db
63
+ .prepare('SELECT markdown,sha256,bytes FROM representations WHERE markdown IS NOT NULL')
64
+ .iterate()) {
65
+ if (typeof row.markdown !== 'string' ||
66
+ sha256(row.markdown) !== row.sha256 ||
67
+ Buffer.byteLength(row.markdown) !== row.bytes)
41
68
  invalid();
42
69
  }
43
- return { schemaVersion: version, libraryId: info.uuid, revisionEpoch: bounded(info.revision_epoch, Number.MAX_SAFE_INTEGER), blobs };
70
+ return {
71
+ schemaVersion: version,
72
+ libraryId: info.uuid,
73
+ revisionEpoch: bounded(info.revision_epoch, Number.MAX_SAFE_INTEGER),
74
+ blobs,
75
+ };
44
76
  }
45
77
  finally {
46
78
  expected.close();
@@ -55,9 +87,11 @@ export function neutralizeRestoredDatabase(path) {
55
87
  try {
56
88
  db.exec('PRAGMA trusted_schema=OFF; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE');
57
89
  // Every nonterminal intent is retained but requires an explicit retry after restore.
58
- const held = db.prepare(`UPDATE jobs SET state='needs-permission',reason='restore-reconciliation-required'${inspected.schemaVersion >= 2 ? ",retryable=0" : ""} WHERE state NOT IN ('succeeded','failed','cancelled','unsupported')`).run();
90
+ const held = db
91
+ .prepare(`UPDATE jobs SET state='needs-permission',reason='restore-reconciliation-required'${inspected.schemaVersion >= 2 ? ',retryable=0' : ''} WHERE state NOT IN ('succeeded','failed','cancelled','unsupported')`)
92
+ .run();
59
93
  db.prepare("UPDATE attempts SET stage='restored-unexecuted',cleanup='settled',finished_at=COALESCE(finished_at,?) WHERE cleanup='pending' OR finished_at IS NULL").run(new Date().toISOString());
60
- db.exec("DELETE FROM staging_pins; DELETE FROM backup_fences; UPDATE worker_owner SET incarnation=NULL,fence=fence+1; UPDATE model_artifacts SET available=0;");
94
+ db.exec('DELETE FROM staging_pins; DELETE FROM backup_fences; UPDATE worker_owner SET incarnation=NULL,fence=fence+1; UPDATE model_artifacts SET available=0;');
61
95
  if (inspected.schemaVersion >= 4) {
62
96
  db.exec("UPDATE index_jobs SET state='failed',reason='restore-reconciliation-required' WHERE state NOT IN ('succeeded','failed','cancelled')");
63
97
  db.prepare("UPDATE index_attempts SET stage='restored-unexecuted',cleanup='settled',finished_at=COALESCE(finished_at,?) WHERE cleanup='pending' OR finished_at IS NULL").run(new Date().toISOString());
@@ -1,31 +1,57 @@
1
1
  import { join } from 'node:path';
2
- import { StoreError } from '../store/types.js';
3
2
  import { safeDirectory } from '../store/files.js';
4
- import { newDirectory, discardOwned, copyVerified, syncDirectory, blobPath, writePrivateExclusive } from './files.js';
3
+ import { newDirectory, discardOwned, copyVerified, blobPath, writePrivateExclusive } from './files.js';
4
+ import { syncDirectory } from '../util/fsync.js';
5
+ import { SpongeError } from '../errors.js';
5
6
  /** Export a fixed revision with hash-named local assets. No network or acquisition. */
6
7
  export function exportItem(store, itemId, output) {
7
8
  const item = store.read(itemId);
8
9
  if (!item)
9
- throw new StoreError('invalid-input');
10
+ throw new SpongeError('invalid-input');
10
11
  const target = newDirectory(output);
11
12
  try {
12
13
  safeDirectory(join(output, 'assets'), true);
13
14
  const known = new Map();
14
15
  for (const asset of item.assets) {
15
16
  if (asset.relativePath !== blobPath(asset.sha256))
16
- throw new StoreError('integrity');
17
+ throw new SpongeError('integrity');
17
18
  const path = `assets/${asset.sha256}`;
18
19
  if (!known.has(asset.sha256)) {
19
- copyVerified(join(store.root, asset.relativePath), join(output, path), { path, bytes: asset.bytes, sha256: asset.sha256 });
20
+ copyVerified(join(store.root, asset.relativePath), join(output, path), {
21
+ path,
22
+ bytes: asset.bytes,
23
+ sha256: asset.sha256,
24
+ });
20
25
  known.set(asset.sha256, path);
21
26
  }
22
27
  }
23
- const markdown = item.markdown.replace(/sponge-asset:([a-f0-9]{64})/gu, (_all, sha) => { const path = known.get(sha); if (!path)
24
- throw new StoreError('integrity'); return path; });
28
+ const markdown = item.markdown.replace(/sponge-asset:([a-f0-9]{64})/gu, (_all, sha) => {
29
+ const path = known.get(sha);
30
+ if (!path)
31
+ throw new SpongeError('integrity');
32
+ return path;
33
+ });
25
34
  writePrivateExclusive(join(output, 'article.md'), markdown);
26
35
  // Metadata can contain private descriptive content; exports stay private and explicit.
27
- const notes = store.readAnnotations(itemId);
28
- writePrivateExclusive(join(output, 'item.json'), JSON.stringify({ schemaVersion: 1, ...notes, itemId: item.itemId, revisionId: item.revisionId, title: item.title, kind: item.kind, status: item.status, metadata: item.metadata, metadataWarnings: item.metadataWarnings, assets: item.assets.map(a => ({ sha256: a.sha256, bytes: a.bytes, mediaType: a.mediaType, relationship: a.relationship, path: known.get(a.sha256) })) }) + '\n');
36
+ const notes = store.annotations.forExport(itemId);
37
+ writePrivateExclusive(join(output, 'item.json'), JSON.stringify({
38
+ schemaVersion: 1,
39
+ ...notes,
40
+ itemId: item.itemId,
41
+ revisionId: item.revisionId,
42
+ title: item.title,
43
+ kind: item.kind,
44
+ status: item.status,
45
+ metadata: item.metadata,
46
+ metadataWarnings: item.metadataWarnings,
47
+ assets: item.assets.map((a) => ({
48
+ sha256: a.sha256,
49
+ bytes: a.bytes,
50
+ mediaType: a.mediaType,
51
+ relationship: a.relationship,
52
+ path: known.get(a.sha256),
53
+ })),
54
+ }) + '\n');
29
55
  syncDirectory(output);
30
56
  return { output, itemId, revisionId: item.revisionId, assets: known.size };
31
57
  }
@@ -1,4 +1,3 @@
1
- import { syncDirectory } from '../store/files.js';
2
1
  export declare const maximumTotalBytes: number, maximumDatabaseBytes: number, maximumBlobBytes: number, maximumEntries = 100000;
3
2
  export interface FileRecord {
4
3
  path: string;
@@ -6,7 +5,7 @@ export interface FileRecord {
6
5
  sha256: string;
7
6
  }
8
7
  export declare function invalid(): never;
9
- export declare function exact(value: unknown, keys: string[]): Record<string, unknown>;
8
+ export declare const exact: (value: unknown, keys: string[]) => Record<string, unknown>;
10
9
  export declare function bounded(value: unknown, max: number): number;
11
10
  export declare function digestString(value: unknown): string;
12
11
  export declare function blobPath(sha: string): string;
@@ -28,6 +27,4 @@ export declare function discardOwned(value: {
28
27
  device: number;
29
28
  inode: number;
30
29
  }): void;
31
- export declare function assertEmptyDirectory(path: string): void;
32
- export { syncDirectory };
33
30
  export declare function writePrivateExclusive(path: string, value: string): void;
@@ -1,78 +1,138 @@
1
1
  import { createHash } from 'node:crypto';
2
- import { constants, openSync, closeSync, fstatSync, readSync, writeSync, fsyncSync, lstatSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
2
+ import { constants, openSync, closeSync, fstatSync, readSync, writeSync, fsyncSync, lstatSync, mkdirSync, rmSync, } from 'node:fs';
3
3
  import { dirname, resolve } from 'node:path';
4
- import { safeDirectory, regularFile, syncDirectory } from '../store/files.js';
5
- import { StoreError } from '../store/types.js';
4
+ import { safeDirectory, regularFile } from '../store/files.js';
5
+ import { syncDirectory } from '../util/fsync.js';
6
+ import { exactKeys } from '../util/exact.js';
7
+ import { SpongeError } from '../errors.js';
6
8
  export const maximumTotalBytes = 16 * 1024 ** 3, maximumDatabaseBytes = 512 * 1024 ** 2, maximumBlobBytes = 512 * 1024 ** 2, maximumEntries = 100_000;
7
- export function invalid() { throw new StoreError('integrity'); }
8
- export function exact(value, keys) { if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).sort().join(',') !== keys.sort().join(','))
9
- invalid(); return value; }
10
- export function bounded(value, max) { if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0 || value > max)
11
- invalid(); return value; }
12
- export function digestString(value) { if (typeof value !== 'string' || !/^[a-f0-9]{64}$/u.test(value))
13
- invalid(); return value; }
14
- export function blobPath(sha) { return `blobs/sha256/${sha.slice(0, 2)}/${sha}`; }
15
- export function fileDescriptor(value) { const r = exact(value, ['path', 'bytes', 'sha256']); const sha256 = digestString(r.sha256); if (r.path !== 'library.sqlite' && r.path !== blobPath(sha256))
16
- invalid(); return { path: String(r.path), bytes: bounded(r.bytes, r.path === 'library.sqlite' ? maximumDatabaseBytes : maximumBlobBytes), sha256 }; }
17
- export function openRead(path) { safeDirectory(dirname(path)); regularFile(path); const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); const stat = fstatSync(fd); if (!stat.isFile() || stat.nlink !== 1) {
18
- closeSync(fd);
19
- invalid();
20
- } return fd; }
21
- export function writeAll(fd, bytes) { let n = 0; while (n < bytes.length) {
22
- const written = writeSync(fd, bytes, n, bytes.length - n);
23
- if (!written)
9
+ export function invalid() {
10
+ throw new SpongeError('integrity');
11
+ }
12
+ export const exact = (value, keys) => exactKeys(value, keys, invalid);
13
+ export function bounded(value, max) {
14
+ if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0 || value > max)
24
15
  invalid();
25
- n += written;
26
- } }
27
- export function readExact(fd, bytes) { const result = Buffer.alloc(bytes); let n = 0; while (n < bytes) {
28
- const read = readSync(fd, result, n, bytes - n, null);
29
- if (!read)
16
+ return value;
17
+ }
18
+ export function digestString(value) {
19
+ if (typeof value !== 'string' || !/^[a-f0-9]{64}$/u.test(value))
30
20
  invalid();
31
- n += read;
32
- } return result; }
33
- /** Stream only expected bytes from one descriptor; never allocate a full PDF or archive. */
34
- export function transfer(input, bytes, output) { const hash = createHash('sha256'), buffer = Buffer.alloc(64 * 1024); let n = 0; while (n < bytes) {
35
- const read = readSync(input, buffer, 0, Math.min(buffer.length, bytes - n), null);
36
- if (!read)
21
+ return value;
22
+ }
23
+ export function blobPath(sha) {
24
+ return `blobs/sha256/${sha.slice(0, 2)}/${sha}`;
25
+ }
26
+ export function fileDescriptor(value) {
27
+ const r = exact(value, ['path', 'bytes', 'sha256']);
28
+ const sha256 = digestString(r.sha256);
29
+ if (r.path !== 'library.sqlite' && r.path !== blobPath(sha256))
37
30
  invalid();
38
- hash.update(buffer.subarray(0, read));
39
- if (output !== undefined)
40
- writeAll(output, buffer.subarray(0, read));
41
- n += read;
42
- } return hash.digest('hex'); }
43
- export function describe(path, relativePath, maximum) { const fd = openRead(path); try {
44
- const bytes = bounded(fstatSync(fd).size, maximum);
45
- return { path: relativePath, bytes, sha256: transfer(fd, bytes) };
31
+ return {
32
+ path: String(r.path),
33
+ bytes: bounded(r.bytes, r.path === 'library.sqlite' ? maximumDatabaseBytes : maximumBlobBytes),
34
+ sha256,
35
+ };
46
36
  }
47
- finally {
48
- closeSync(fd);
49
- } }
50
- export function copyVerified(source, target, record) { const input = openRead(source); let output; try {
51
- if (fstatSync(input).size !== record.bytes)
37
+ export function openRead(path) {
38
+ safeDirectory(dirname(path));
39
+ regularFile(path);
40
+ const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
41
+ const stat = fstatSync(fd);
42
+ if (!stat.isFile() || stat.nlink !== 1) {
43
+ closeSync(fd);
52
44
  invalid();
53
- safeDirectory(dirname(target), true);
54
- output = openSync(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
55
- if (transfer(input, record.bytes, output) !== record.sha256)
45
+ }
46
+ return fd;
47
+ }
48
+ export function writeAll(fd, bytes) {
49
+ let n = 0;
50
+ while (n < bytes.length) {
51
+ const written = writeSync(fd, bytes, n, bytes.length - n);
52
+ if (!written)
53
+ invalid();
54
+ n += written;
55
+ }
56
+ }
57
+ export function readExact(fd, bytes) {
58
+ const result = Buffer.alloc(bytes);
59
+ let n = 0;
60
+ while (n < bytes) {
61
+ const read = readSync(fd, result, n, bytes - n, null);
62
+ if (!read)
63
+ invalid();
64
+ n += read;
65
+ }
66
+ return result;
67
+ }
68
+ /** Stream only expected bytes from one descriptor; never allocate a full PDF or archive. */
69
+ export function transfer(input, bytes, output) {
70
+ const hash = createHash('sha256'), buffer = Buffer.alloc(64 * 1024);
71
+ let n = 0;
72
+ while (n < bytes) {
73
+ const read = readSync(input, buffer, 0, Math.min(buffer.length, bytes - n), null);
74
+ if (!read)
75
+ invalid();
76
+ hash.update(buffer.subarray(0, read));
77
+ if (output !== undefined)
78
+ writeAll(output, buffer.subarray(0, read));
79
+ n += read;
80
+ }
81
+ return hash.digest('hex');
82
+ }
83
+ export function describe(path, relativePath, maximum) {
84
+ const fd = openRead(path);
85
+ try {
86
+ const bytes = bounded(fstatSync(fd).size, maximum);
87
+ return { path: relativePath, bytes, sha256: transfer(fd, bytes) };
88
+ }
89
+ finally {
90
+ closeSync(fd);
91
+ }
92
+ }
93
+ export function copyVerified(source, target, record) {
94
+ const input = openRead(source);
95
+ let output;
96
+ try {
97
+ if (fstatSync(input).size !== record.bytes)
98
+ invalid();
99
+ safeDirectory(dirname(target), true);
100
+ output = openSync(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
101
+ if (transfer(input, record.bytes, output) !== record.sha256)
102
+ invalid();
103
+ fsyncSync(output);
104
+ syncDirectory(dirname(target));
105
+ }
106
+ finally {
107
+ closeSync(input);
108
+ if (output !== undefined)
109
+ closeSync(output);
110
+ }
111
+ }
112
+ export function newDirectory(path) {
113
+ if (resolve(path) !== path)
114
+ throw new SpongeError('invalid-input');
115
+ safeDirectory(dirname(path));
116
+ mkdirSync(path, { mode: 0o700 });
117
+ const stat = lstatSync(path);
118
+ return { root: path, device: stat.dev, inode: stat.ino };
119
+ }
120
+ export function discardOwned(value) {
121
+ const stat = lstatSync(value.root);
122
+ if (!stat.isDirectory() || stat.isSymbolicLink() || stat.dev !== value.device || stat.ino !== value.inode)
56
123
  invalid();
57
- fsyncSync(output);
58
- syncDirectory(dirname(target));
124
+ rmSync(value.root, { recursive: true });
125
+ syncDirectory(dirname(value.root));
59
126
  }
60
- finally {
61
- closeSync(input);
62
- if (output !== undefined)
63
- closeSync(output);
64
- } }
65
- export function newDirectory(path) { if (resolve(path) !== path)
66
- throw new StoreError('invalid-input'); safeDirectory(dirname(path)); mkdirSync(path, { mode: 0o700 }); const stat = lstatSync(path); return { root: path, device: stat.dev, inode: stat.ino }; }
67
- export function discardOwned(value) { const stat = lstatSync(value.root); if (!stat.isDirectory() || stat.isSymbolicLink() || stat.dev !== value.device || stat.ino !== value.inode)
68
- invalid(); rmSync(value.root, { recursive: true }); syncDirectory(dirname(value.root)); }
69
- export function assertEmptyDirectory(path) { safeDirectory(path); if (readdirSync(path).length)
70
- invalid(); }
71
- export { syncDirectory };
72
- export function writePrivateExclusive(path, value) { safeDirectory(dirname(path)); const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); try {
73
- writeAll(fd, Buffer.from(value));
74
- fsyncSync(fd);
127
+ export function writePrivateExclusive(path, value) {
128
+ safeDirectory(dirname(path));
129
+ const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
130
+ try {
131
+ writeAll(fd, Buffer.from(value));
132
+ fsyncSync(fd);
133
+ }
134
+ finally {
135
+ closeSync(fd);
136
+ }
137
+ syncDirectory(dirname(path));
75
138
  }
76
- finally {
77
- closeSync(fd);
78
- } syncDirectory(dirname(path)); }