@hraness/sponge 0.2.6 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +21 -9
  2. package/bin/sponge.js +8 -7
  3. package/dist/contracts/index.js +24 -6
  4. package/dist/contracts/validate.js +106 -29
  5. package/dist/migrations/008.d.ts +3 -0
  6. package/dist/migrations/008.js +5 -0
  7. package/dist/migrations/009.d.ts +3 -0
  8. package/dist/migrations/009.js +8 -0
  9. package/dist/migrations/010.d.ts +3 -0
  10. package/dist/migrations/010.js +11 -0
  11. package/dist/migrations/index.d.ts +15 -0
  12. package/dist/migrations/index.js +39 -0
  13. package/dist/src/acquisition/capture.js +87 -28
  14. package/dist/src/acquisition/cleanup.d.ts +0 -4
  15. package/dist/src/acquisition/cleanup.js +3 -5
  16. package/dist/src/acquisition/errors.js +17 -2
  17. package/dist/src/acquisition/files.d.ts +0 -1
  18. package/dist/src/acquisition/files.js +14 -11
  19. package/dist/src/acquisition/index.d.ts +2 -3
  20. package/dist/src/acquisition/index.js +1 -2
  21. package/dist/src/acquisition/lock.js +1 -1
  22. package/dist/src/acquisition/normalize.js +57 -18
  23. package/dist/src/acquisition/pins.js +22 -4
  24. package/dist/src/acquisition/process.js +67 -29
  25. package/dist/src/acquisition/runtime.d.ts +5 -7
  26. package/dist/src/acquisition/runtime.js +129 -68
  27. package/dist/src/acquisition/verify-helper.js +10 -3
  28. package/dist/src/annotations/quote.js +40 -12
  29. package/dist/src/annotations/store.d.ts +0 -1
  30. package/dist/src/annotations/store.js +66 -14
  31. package/dist/src/app/article.d.ts +2 -1
  32. package/dist/src/app/article.js +66 -70
  33. package/dist/src/app/bundle-files.d.ts +0 -4
  34. package/dist/src/app/bundle-files.js +11 -14
  35. package/dist/src/app/markdown.js +18 -6
  36. package/dist/src/app/source.d.ts +0 -4
  37. package/dist/src/app/source.js +18 -14
  38. package/dist/src/cli/arguments.d.ts +2 -8
  39. package/dist/src/cli/arguments.js +67 -80
  40. package/dist/src/cli/commands.d.ts +42 -0
  41. package/dist/src/cli/commands.js +795 -0
  42. package/dist/src/cli/config.js +12 -9
  43. package/dist/src/cli/digest.d.ts +1 -1
  44. package/dist/src/cli/digest.js +38 -16
  45. package/dist/src/cli/import.js +25 -4
  46. package/dist/src/cli/indexing.d.ts +2 -0
  47. package/dist/src/cli/indexing.js +26 -9
  48. package/dist/src/cli/main.js +47 -386
  49. package/dist/src/cli/portability.d.ts +0 -2
  50. package/dist/src/cli/portability.js +13 -8
  51. package/dist/src/cli/queue.js +12 -8
  52. package/dist/src/cli/runtimes.d.ts +1 -3
  53. package/dist/src/cli/runtimes.js +13 -6
  54. package/dist/src/cli/terminal.js +1 -1
  55. package/dist/src/cli/worker.d.ts +7 -0
  56. package/dist/src/cli/worker.js +54 -15
  57. package/dist/src/derivations/kinds.d.ts +34 -0
  58. package/dist/src/derivations/kinds.js +14 -0
  59. package/dist/src/derivations/types.d.ts +48 -0
  60. package/dist/src/derivations/types.js +1 -0
  61. package/dist/src/digest/derivation.d.ts +25 -0
  62. package/dist/src/digest/derivation.js +124 -0
  63. package/dist/src/digest/editorial.d.ts +9 -46
  64. package/dist/src/digest/editorial.js +57 -49
  65. package/dist/src/digest/endpoint.js +63 -27
  66. package/dist/src/digest/pipeline.d.ts +49 -28
  67. package/dist/src/digest/pipeline.js +343 -100
  68. package/dist/src/digest/schema.d.ts +11 -25
  69. package/dist/src/digest/schema.js +86 -18
  70. package/dist/src/errors.d.ts +46 -0
  71. package/dist/src/errors.js +117 -0
  72. package/dist/src/identity/query-policy.d.ts +0 -5
  73. package/dist/src/identity/query-policy.js +61 -9
  74. package/dist/src/identity/uri.d.ts +6 -5
  75. package/dist/src/identity/uri.js +37 -18
  76. package/dist/src/identity/visibility.d.ts +11 -1
  77. package/dist/src/identity/visibility.js +81 -17
  78. package/dist/src/importers/apply.js +104 -36
  79. package/dist/src/importers/custody.js +27 -9
  80. package/dist/src/importers/inspect.js +125 -38
  81. package/dist/src/importers/jungle-digest.d.ts +3 -6
  82. package/dist/src/importers/jungle-digest.js +196 -69
  83. package/dist/src/importers/jungle-store.d.ts +15 -6
  84. package/dist/src/importers/jungle-store.js +122 -28
  85. package/dist/src/importers/jungle.d.ts +14 -14
  86. package/dist/src/importers/jungle.js +136 -31
  87. package/dist/src/importers/manifest.js +57 -21
  88. package/dist/src/importers/verify-helper.js +18 -4
  89. package/dist/src/indexing/image.js +29 -10
  90. package/dist/src/indexing/index.d.ts +37 -6
  91. package/dist/src/indexing/index.js +360 -124
  92. package/dist/src/indexing/jobs.js +156 -47
  93. package/dist/src/indexing/prune-run.d.ts +2 -0
  94. package/dist/src/indexing/prune-run.js +69 -26
  95. package/dist/src/indexing/prune.d.ts +12 -3
  96. package/dist/src/indexing/prune.js +85 -35
  97. package/dist/src/indexing/runner.d.ts +2 -1
  98. package/dist/src/indexing/runner.js +83 -40
  99. package/dist/src/indexing/upgrade.d.ts +9 -0
  100. package/dist/src/indexing/upgrade.js +79 -0
  101. package/dist/src/inference/asr/bridge.js +6 -2
  102. package/dist/src/inference/asr/protocol.js +109 -19
  103. package/dist/src/inference/asr/runtime.js +58 -21
  104. package/dist/src/inference/asr/windows.js +9 -5
  105. package/dist/src/inference/embedding/index.d.ts +2 -6
  106. package/dist/src/inference/embedding/index.js +89 -29
  107. package/dist/src/ingest/attempt.d.ts +29 -0
  108. package/dist/src/ingest/attempt.js +55 -0
  109. package/dist/src/media/capture-helper.js +7 -2
  110. package/dist/src/media/capture.d.ts +2 -1
  111. package/dist/src/media/capture.js +124 -43
  112. package/dist/src/media/handler.d.ts +4 -2
  113. package/dist/src/media/handler.js +88 -70
  114. package/dist/src/media/transcribe.js +63 -21
  115. package/dist/src/metadata/article.js +49 -17
  116. package/dist/src/models/artifacts.d.ts +10 -8
  117. package/dist/src/models/artifacts.js +237 -106
  118. package/dist/src/models/catalog.d.ts +2 -5
  119. package/dist/src/models/catalog.js +94 -25
  120. package/dist/src/models/download.js +55 -15
  121. package/dist/src/models/runtime.d.ts +0 -2
  122. package/dist/src/models/runtime.js +69 -33
  123. package/dist/src/pdf/capture-helper.js +3 -1
  124. package/dist/src/pdf/capture.d.ts +2 -1
  125. package/dist/src/pdf/capture.js +95 -27
  126. package/dist/src/pdf/errors.js +12 -1
  127. package/dist/src/pdf/handler.d.ts +2 -1
  128. package/dist/src/pdf/handler.js +70 -71
  129. package/dist/src/pdf/index.d.ts +1 -3
  130. package/dist/src/pdf/index.js +0 -2
  131. package/dist/src/pdf/input.js +12 -7
  132. package/dist/src/pdf/normalize.js +80 -21
  133. package/dist/src/pdf/source-identity.js +3 -2
  134. package/dist/src/pdf/source.js +12 -8
  135. package/dist/src/pdf/verify-helper.js +15 -3
  136. package/dist/src/portability/backup.d.ts +2 -9
  137. package/dist/src/portability/backup.js +106 -22
  138. package/dist/src/portability/database.js +51 -17
  139. package/dist/src/portability/export.js +35 -9
  140. package/dist/src/portability/files.d.ts +1 -4
  141. package/dist/src/portability/files.js +127 -67
  142. package/dist/src/portability/gc.js +65 -22
  143. package/dist/src/publish/component.js +1 -1
  144. package/dist/src/publish/feed.d.ts +2 -2
  145. package/dist/src/publish/feed.js +75 -29
  146. package/dist/src/publish/identity.d.ts +5 -1
  147. package/dist/src/publish/identity.js +31 -11
  148. package/dist/src/publish/index.d.ts +2 -3
  149. package/dist/src/publish/index.js +34 -21
  150. package/dist/src/publish/projection.d.ts +2 -9
  151. package/dist/src/publish/projection.js +21 -14
  152. package/dist/src/search/extension.js +3 -1
  153. package/dist/src/search/hybrid.d.ts +3 -8
  154. package/dist/src/search/hybrid.js +45 -11
  155. package/dist/src/search/lexical.js +18 -4
  156. package/dist/src/search/query.js +56 -25
  157. package/dist/src/search/vector.js +37 -10
  158. package/dist/src/service/launch-agent.d.ts +5 -3
  159. package/dist/src/service/launch-agent.js +272 -83
  160. package/dist/src/service/lifetime.js +47 -14
  161. package/dist/src/service/loop.d.ts +3 -3
  162. package/dist/src/service/loop.js +47 -19
  163. package/dist/src/store/annotations.d.ts +32 -0
  164. package/dist/src/store/annotations.js +44 -0
  165. package/dist/src/store/backups.d.ts +18 -0
  166. package/dist/src/store/backups.js +110 -0
  167. package/dist/src/store/blobs.d.ts +24 -0
  168. package/dist/src/store/blobs.js +139 -0
  169. package/dist/src/store/changes.d.ts +5 -12
  170. package/dist/src/store/changes.js +108 -56
  171. package/dist/src/store/context.d.ts +15 -0
  172. package/dist/src/store/context.js +19 -0
  173. package/dist/src/store/derivations.d.ts +42 -0
  174. package/dist/src/store/derivations.js +104 -0
  175. package/dist/src/store/files.d.ts +0 -2
  176. package/dist/src/store/files.js +19 -24
  177. package/dist/src/store/ids.d.ts +30 -0
  178. package/dist/src/store/ids.js +34 -0
  179. package/dist/src/store/imports.d.ts +26 -0
  180. package/dist/src/store/imports.js +57 -0
  181. package/dist/src/store/index.d.ts +3 -186
  182. package/dist/src/store/index.js +2 -809
  183. package/dist/src/store/lease.d.ts +20 -0
  184. package/dist/src/store/lease.js +62 -0
  185. package/dist/src/store/library.d.ts +75 -0
  186. package/dist/src/store/library.js +174 -0
  187. package/dist/src/store/open.d.ts +7 -0
  188. package/dist/src/store/open.js +183 -0
  189. package/dist/src/store/queue.d.ts +65 -0
  190. package/dist/src/store/queue.js +327 -0
  191. package/dist/src/store/revisions.d.ts +12 -0
  192. package/dist/src/store/revisions.js +189 -0
  193. package/dist/src/store/search-index.d.ts +63 -0
  194. package/dist/src/store/search-index.js +190 -0
  195. package/dist/src/store/sync-ready.d.ts +6 -0
  196. package/dist/src/store/sync-ready.js +121 -0
  197. package/dist/src/store/types.d.ts +1 -5
  198. package/dist/src/store/types.js +1 -8
  199. package/dist/src/util/canonical.d.ts +2 -0
  200. package/dist/src/util/canonical.js +17 -0
  201. package/dist/src/util/exact.d.ts +2 -0
  202. package/dist/src/util/exact.js +9 -0
  203. package/dist/src/util/fsync.d.ts +4 -0
  204. package/dist/src/util/fsync.js +22 -0
  205. package/dist/src/util/hash.d.ts +4 -0
  206. package/dist/src/util/hash.js +8 -0
  207. package/dist/src/worker/basic.d.ts +0 -1
  208. package/dist/src/worker/basic.js +23 -7
  209. package/dist/src/worker/custody.d.ts +4 -1
  210. package/dist/src/worker/custody.js +111 -38
  211. package/dist/src/worker/engine.d.ts +2 -1
  212. package/dist/src/worker/engine.js +42 -20
  213. package/dist/src/worker/identity.js +13 -5
  214. package/dist/src/worker/journal.d.ts +1 -1
  215. package/dist/src/worker/journal.js +54 -24
  216. package/dist/src/worker/lock.d.ts +18 -0
  217. package/dist/src/worker/lock.js +196 -0
  218. package/dist/src/worker/runtime-gc.d.ts +4 -3
  219. package/dist/src/worker/runtime-gc.js +68 -41
  220. package/dist/src/worker/supervisor-child.js +19 -6
  221. package/dist/src/worker/supervisor.js +41 -8
  222. package/native/runtime-manifest.json +4 -4
  223. package/package.json +10 -2
  224. package/dist/src/digest/store.d.ts +0 -57
  225. package/dist/src/digest/store.js +0 -79
  226. package/dist/src/store/foundation.d.ts +0 -7
  227. package/dist/src/store/foundation.js +0 -7
@@ -1,810 +1,3 @@
1
- import { DatabaseSync, backup } from 'node:sqlite';
2
- import { chmodSync, closeSync, constants, readFileSync, fstatSync, readdirSync, fsyncSync, linkSync, mkdirSync, openSync, writeFileSync, rmSync, statfsSync, unlinkSync, writeSync } from 'node:fs';
3
- import { join, dirname, parse } from 'node:path';
4
- import { randomUUID, createHash } from 'node:crypto';
5
- import { sql as sql1 } from '../../migrations/001.js';
6
- import { sql as sql2 } from '../../migrations/002.js';
7
- import { sql as sql3 } from '../../migrations/003.js';
8
- import { sql as sql4 } from '../../migrations/004.js';
9
- import { sql as sql5 } from '../../migrations/005.js';
10
- import { sql as sql6 } from '../../migrations/006.js';
11
- import { sql as sql7, version } from '../../migrations/007.js';
12
- import * as foundation from './foundation.js';
13
- import * as jungle from '../importers/jungle-store.js';
14
- import * as indexJobs from '../indexing/jobs.js';
15
- import { limits } from '../../contracts/index.js';
16
- import { validateMetadataClaim, validateMetadataWarnings } from '../../contracts/validate.js';
17
- import { confinedFile, digest, hash, integer, regularFile, safeDirectory, syncDirectory, text, verifyFile, pathExists, readVerifiedFile } from './files.js';
18
- import { StoreError } from './types.js';
19
- import * as indexing from '../indexing/index.js';
20
- import * as prune from '../indexing/prune.js';
21
- import { hybridSearch } from '../search/hybrid.js';
22
- import { indexReadable, unindexReadable, lexicalSearch } from '../search/lexical.js';
23
- import { initializeVectorExtension } from '../search/extension.js';
1
+ export { LibraryStore } from './library.js';
2
+ export { openLibrary, keptSchemaBackups, libraryVersion } from './open.js';
24
3
  export * from './types.js';
25
- const now = () => new Date().toISOString();
26
- const migrations = [sql1, sql2, sql3, sql4, sql5, sql6, sql7];
27
- export const libraryVersion = version;
28
- export const libraryMigrations = Object.freeze([...migrations]);
29
- export async function openLibrary(path, options = {}) {
30
- if (path === parse(path).root)
31
- throw new StoreError('invalid-input');
32
- const root = safeDirectory(path, true);
33
- if (root === parse(root).root)
34
- throw new StoreError('invalid-input');
35
- const marker = join(root, '.sponge-library'), dbPath = confinedFile(root, join(root, 'library.sqlite'));
36
- if (pathExists(marker)) {
37
- regularFile(marker);
38
- verifyFile(marker, digest('sponge-library-v1\n'), Buffer.byteLength('sponge-library-v1\n'));
39
- }
40
- else if (readdirSync(root).length)
41
- throw new StoreError('invalid-input');
42
- for (const suffix of ['', '-wal', '-shm'])
43
- if (pathExists(dbPath + suffix))
44
- regularFile(dbPath + suffix);
45
- if (pathExists(dbPath)) {
46
- const inspect = new DatabaseSync(dbPath, { readOnly: true });
47
- try {
48
- if (Number(inspect.prepare('PRAGMA user_version').get()?.user_version) > version)
49
- throw new StoreError('schema-newer');
50
- }
51
- finally {
52
- inspect.close();
53
- }
54
- }
55
- chmodSync(root, 0o700);
56
- if (!pathExists(marker)) {
57
- writeFileSync(marker, 'sponge-library-v1\n', { flag: 'wx', mode: 0o600 });
58
- const fd = openSync(marker, constants.O_RDONLY | constants.O_NOFOLLOW);
59
- try {
60
- fsyncSync(fd);
61
- }
62
- finally {
63
- closeSync(fd);
64
- }
65
- syncDirectory(root);
66
- }
67
- for (const part of ['runtime', 'staging', 'blobs', 'blobs/sha256', 'backups']) {
68
- const p = safeDirectory(join(root, part), true);
69
- chmodSync(p, 0o700);
70
- }
71
- const db = new DatabaseSync(dbPath, { allowExtension: true });
72
- chmodSync(dbPath, 0o600);
73
- try {
74
- db.exec('PRAGMA foreign_keys=ON; PRAGMA busy_timeout=5000;');
75
- const current = Number(db.prepare('PRAGMA user_version').get()?.user_version);
76
- if (current > version)
77
- throw new StoreError('schema-newer');
78
- if (current < version) {
79
- const lock = join(root, 'runtime', 'migration.lock');
80
- try {
81
- mkdirSync(lock, { mode: 0o700 });
82
- }
83
- catch (error) {
84
- if (error.code === 'EEXIST')
85
- throw new StoreError('owner-busy');
86
- throw error;
87
- }
88
- try {
89
- // Recheck under the migration lock: another opener may already have migrated.
90
- const actual = Number(db.prepare('PRAGMA user_version').get()?.user_version);
91
- if (actual > version)
92
- throw new StoreError('schema-newer');
93
- if (actual === 0 && db.prepare("SELECT name FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'").all().length)
94
- throw new StoreError('integrity');
95
- for (let prior = 1; prior <= actual; prior++) {
96
- if (db.prepare('SELECT checksum FROM schema_migrations WHERE version=?').get(prior)?.checksum !== digest(migrations[prior - 1]))
97
- throw new StoreError('integrity');
98
- }
99
- if (actual > 0 && db.prepare('SELECT schema_version FROM library_info WHERE id=1').get()?.schema_version !== actual)
100
- throw new StoreError('integrity');
101
- let backupPath;
102
- try {
103
- if (actual < version) {
104
- // SQLite backup incorporates WAL; never copy the main file alone.
105
- backupPath = join(root, 'backups', `before-schema-${version}-${randomUUID()}.sqlite`);
106
- await backup(db, backupPath);
107
- chmodSync(backupPath, 0o600);
108
- const fd = openSync(backupPath, constants.O_RDONLY);
109
- try {
110
- fsyncSync(fd);
111
- }
112
- finally {
113
- closeSync(fd);
114
- }
115
- syncDirectory(dirname(backupPath));
116
- }
117
- for (let target = actual + 1; target <= version; target++) {
118
- db.exec('BEGIN IMMEDIATE');
119
- try {
120
- db.exec(migrations[target - 1]);
121
- // Code steps are total per item: one hostile row never makes the library unopenable.
122
- if (target === 5) {
123
- foundation.backfillUris(db);
124
- foundation.seedChanges(db);
125
- }
126
- if (target === 1)
127
- db.prepare('INSERT INTO library_info(id,uuid,schema_version,created_by) VALUES(1,?,?,?)').run(randomUUID(), target, options.appVersion ?? 'development');
128
- else
129
- db.prepare('UPDATE library_info SET schema_version=? WHERE id=1').run(target);
130
- db.prepare('INSERT INTO schema_migrations VALUES(?,?,?)').run(target, digest(migrations[target - 1]), now());
131
- db.exec(`PRAGMA user_version=${target}; COMMIT`);
132
- }
133
- catch (error) {
134
- db.exec('ROLLBACK');
135
- throw error;
136
- }
137
- }
138
- }
139
- catch (error) {
140
- // Nothing committed, so the library is byte-for-byte what the backup holds: drop the copy.
141
- if (backupPath && Number(db.prepare('PRAGMA user_version').get()?.user_version) === actual) {
142
- rmSync(backupPath, { force: true });
143
- syncDirectory(dirname(backupPath));
144
- }
145
- throw error;
146
- }
147
- }
148
- finally {
149
- rmSync(lock, { recursive: true });
150
- syncDirectory(join(root, 'runtime'));
151
- }
152
- }
153
- for (let applied = 1; applied <= version; applied++)
154
- if (db.prepare('SELECT checksum FROM schema_migrations WHERE version=?').get(applied)?.checksum !== digest(migrations[applied - 1]))
155
- throw new StoreError('integrity');
156
- if (db.prepare('SELECT schema_version FROM library_info WHERE id=1').get()?.schema_version !== version)
157
- throw new StoreError('integrity');
158
- db.exec('PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;');
159
- for (const suffix of ['', '-wal', '-shm'])
160
- if (pathExists(dbPath + suffix))
161
- chmodSync(dbPath + suffix, 0o600);
162
- syncDirectory(root);
163
- initializeVectorExtension(db);
164
- return new LibraryStore(root, db, options);
165
- }
166
- catch (error) {
167
- db.close();
168
- throw error;
169
- }
170
- }
171
- /** Storage CAS authority only. The worker separately establishes OS/process custody. */
172
- export class LibraryStore {
173
- root;
174
- db;
175
- options;
176
- closed = false;
177
- constructor(root, db, options) {
178
- this.root = root;
179
- this.db = db;
180
- this.options = options;
181
- }
182
- createIndexGeneration(options) { return indexing.createGeneration(this.db, options); }
183
- submitIndexJob(generationId) { return indexJobs.submitIndexJob(this.db, generationId); }
184
- indexJob(id) { return indexJobs.indexJob(this.db, id); }
185
- claimIndexJob(owner, id) { return indexJobs.claimIndexJob(this.db, owner, id); }
186
- checkpointIndexAttempt(token, stage) { indexJobs.checkpointIndexAttempt(this.db, token, stage); }
187
- writeIndexAttemptBatch(token, fingerprint, batch) { indexJobs.withIndexAttempt(this.db, token, id => indexing.writeBatch(this.db, id, fingerprint, batch)); }
188
- completeIndexAttempt(token) { return indexJobs.withIndexAttempt(this.db, token, id => { const status = indexing.activateGeneration(this.db, id); indexJobs.finishIndexAttempt(this.db, token); return status; }); }
189
- failIndexAttempt(token, reason) { indexJobs.finishIndexAttempt(this.db, token, reason); }
190
- settleIndexAttempt(token) { this.transaction(() => { this.checkOwner(token); const a = this.db.prepare('SELECT * FROM index_attempts WHERE id=? AND job_id=?').get(token.attemptId, token.jobId); if (!a || a.owner_incarnation !== token.ownerIncarnation || a.fence !== token.fence || a.finished_at === null)
191
- throw new StoreError('owner-busy'); const path = join(this.root, 'staging', token.attemptId); if (pathExists(path)) {
192
- this.checkIndexStaging(token, path);
193
- rmSync(path, { recursive: true });
194
- syncDirectory(dirname(path));
195
- } indexJobs.settleIndexAttempt(this.db, token); }); }
196
- checkIndexStaging(token, path) { safeDirectory(path); const fd = openSync(join(path, 'ownership.json'), constants.O_RDONLY | constants.O_NOFOLLOW); try {
197
- const stat = fstatSync(fd);
198
- if (!stat.isFile() || stat.size > 65536 || (stat.mode & 0o077) !== 0)
199
- throw new StoreError('integrity');
200
- const marker = JSON.parse(readFileSync(fd, 'utf8'));
201
- if (marker.schemaVersion !== 1 || marker.requestId !== token.attemptId || marker.ownerIncarnation !== token.ownerIncarnation)
202
- throw new StoreError('integrity');
203
- }
204
- finally {
205
- closeSync(fd);
206
- } }
207
- indexStatus(id) { return indexing.generationStatus(this.db, id); }
208
- indexInputs(id, limit = 16) { return indexing.pendingInputs(this.db, id, limit); }
209
- writeIndexBatch(id, fingerprint, batch) { indexing.writeBatch(this.db, id, fingerprint, batch); }
210
- activateIndex(id) { return indexing.activateGeneration(this.db, id); }
211
- cancelIndex(id) { indexing.cancelGeneration(this.db, id); }
212
- /** Retired-generation pruning; see src/indexing/prune.ts. The plan probes rollback inside a rolled-back savepoint. */
213
- pruneIndexPlan(keep) { return this.transaction(() => ({ generations: prune.planPrune(this.db, keep), stats: prune.pageStats(this.db) })); }
214
- pruneIndexStep(owner, id, keep) { text(id, 256); return this.transaction(() => { this.checkOwner(owner); if (this.pendingCleanupCount() || this.hasGcFence())
215
- throw new StoreError('owner-busy'); return prune.pruneStep(this.db, id, keep); }); }
216
- /** VACUUM cannot run in a transaction: fences are checked under the owner token immediately before it. */
217
- vacuumLibrary(owner) { const stats = this.transaction(() => { this.checkOwner(owner); if (this.pendingCleanupCount() || this.hasGcFence())
218
- throw new StoreError('owner-busy'); return prune.pageStats(this.db); }); prune.vacuumSpace(this.root, stats.bytes); this.db.exec('VACUUM'); this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); return prune.pageStats(this.db); }
219
- searchHybrid(query, options = {}) { return hybridSearch(this.db, query, options); }
220
- /** Export's view: the one annotation reader, without the redundant item id, plus reading state. */
221
- readAnnotations(itemId) { text(itemId, 256); const annotations = foundation.listAnnotations(this.db, itemId).map(({ itemId: _, ...a }) => a); const reading = this.db.prepare('SELECT state_json,updated_at FROM reading_state WHERE item_id=?').get(itemId); return { annotations, readingState: reading ? { state: JSON.parse(String(reading.state_json)), updatedAt: String(reading.updated_at) } : null }; }
222
- /** Sync and publishing foundation; see docs/decisions/sync-and-publishing.md. */
223
- identity(itemId) { text(itemId, 256); return { uris: foundation.itemUris(this.db, itemId), visibility: foundation.visibility(this.db, itemId) }; }
224
- itemsForUri(uri) { return foundation.itemsForUri(this.db, uri); }
225
- setVisibility(itemId, value) { text(itemId, 256); return this.transaction(() => foundation.setVisibility(this.db, itemId, value)); }
226
- addHighlight(input) { text(input.itemId, 256); return this.transaction(() => foundation.addHighlight(this.db, input)); }
227
- addComment(input) { text(input.itemId, 256); if (input.parentId !== undefined)
228
- text(input.parentId, 256); return this.transaction(() => foundation.addComment(this.db, input)); }
229
- updateAnnotation(id, body) { text(id, 256); return this.transaction(() => foundation.updateAnnotation(this.db, id, body)); }
230
- deleteAnnotation(id) { text(id, 256); this.transaction(() => foundation.deleteAnnotation(this.db, id)); }
231
- listAnnotations(itemId) { text(itemId, 256); return foundation.listAnnotations(this.db, itemId); }
232
- putDigest(input) { text(input.itemId, 256); text(input.revisionId, 256); return this.transaction(() => foundation.putDigest(this.db, input)); }
233
- currentDigest(itemId) { text(itemId, 256); return foundation.currentDigest(this.db, itemId); }
234
- readDigest(id) { text(id, 256); return foundation.readDigest(this.db, id); }
235
- findDigest(revisionId, inputHash) { text(revisionId, 256); return foundation.findDigest(this.db, revisionId, inputHash); }
236
- reviewDigest(id, review) { text(id, 256); return this.transaction(() => foundation.reviewDigest(this.db, id, review)); }
237
- currentEditorial(itemId) { text(itemId, 256); return foundation.currentEditorial(this.db, itemId); }
238
- readEditorial(id) { text(id, 256); return foundation.readEditorial(this.db, id); }
239
- reviewEditorial(id, status) { text(id, 256); return this.transaction(() => foundation.reviewEditorial(this.db, id, status)); }
240
- /** The item an importer realm holds for one source key, with its current revision; deleted items are reported, not hidden. */
241
- importedItem(realm, sourceKey) { text(realm, 256); text(sourceKey, 8192); const r = this.db.prepare('SELECT i.id,i.deleted_at,i.current_revision FROM source_refs s JOIN items i ON i.id=s.item_id WHERE s.source_key=? AND s.realm_key=?').get(sourceKey, realm); return r ? { itemId: String(r.id), deleted: r.deleted_at !== null, revisionId: r.current_revision === null ? null : String(r.current_revision) } : null; }
242
- importedRevision(itemId, bundleHash) { text(itemId, 256); hash(bundleHash); const r = this.db.prepare('SELECT id FROM revisions WHERE item_id=? AND bundle_hash=?').get(itemId, bundleHash); if (!r)
243
- throw new StoreError('unavailable'); return String(r.id); }
244
- revisionMarkdown(revisionId) { text(revisionId, 256); return foundation.markdownFor(this.db, revisionId); }
245
- planJungleNote(input) { text(input.itemId, 256); text(input.revisionId, 256); return jungle.planJungleNote(this.db, input); }
246
- applyJungleNote(input) { text(input.itemId, 256); text(input.revisionId, 256); return this.transaction(() => jungle.applyJungleNote(this.db, input)); }
247
- listChanges(after = 0, limit = 500) { return foundation.listChanges(this.db, after, limit); }
248
- publicProjection() { return foundation.publicProjection(this.db); }
249
- lookupImportReceipt(namespace, sourceKey, sourceHash) {
250
- text(namespace, 256);
251
- text(sourceKey, 8192);
252
- hash(sourceHash);
253
- const row = this.db.prepare('SELECT item_id itemId,outcome FROM import_receipts WHERE namespace=? AND source_key=? AND source_hash=?').get(namespace, sourceKey, sourceHash);
254
- return row ? { itemId: String(row.itemId), outcome: String(row.outcome) } : null;
255
- }
256
- close() {
257
- if (!this.closed) {
258
- this.db.close();
259
- this.closed = true;
260
- }
261
- }
262
- transaction(fn) {
263
- this.db.exec('BEGIN IMMEDIATE');
264
- try {
265
- const result = fn();
266
- this.db.exec('COMMIT');
267
- return result;
268
- }
269
- catch (error) {
270
- this.db.exec('ROLLBACK');
271
- throw error;
272
- }
273
- }
274
- get revisionEpoch() { return Number(this.db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch); }
275
- bumpEpoch() { this.db.exec('UPDATE library_info SET revision_epoch=revision_epoch+1 WHERE id=1'); }
276
- acquireOwner(ownerIncarnation) {
277
- text(ownerIncarnation, 256);
278
- return this.transaction(() => {
279
- const old = this.db.prepare('SELECT * FROM worker_owner WHERE id=1').get();
280
- if (old.incarnation !== null && old.incarnation !== ownerIncarnation)
281
- throw new StoreError('owner-busy');
282
- if (old.incarnation === ownerIncarnation)
283
- return { ownerIncarnation, fence: Number(old.fence) };
284
- this.db.prepare('UPDATE worker_owner SET incarnation=?,fence=fence+1 WHERE id=1').run(ownerIncarnation);
285
- return { ownerIncarnation, fence: Number(old.fence) + 1 };
286
- });
287
- }
288
- releaseOwner(token) {
289
- this.transaction(() => {
290
- this.checkOwner(token);
291
- if (this.ownerAttempts(token).some(attempt => attempt.cleanup === 'pending'))
292
- throw new StoreError('owner-busy');
293
- this.db.prepare('UPDATE worker_owner SET incarnation=NULL WHERE id=1').run();
294
- });
295
- }
296
- ownerSnapshot() {
297
- const row = this.db.prepare('SELECT incarnation,fence FROM worker_owner WHERE id=1').get();
298
- return row.incarnation === null ? null : { ownerIncarnation: String(row.incarnation), fence: Number(row.fence) };
299
- }
300
- ownerAttempts(token) {
301
- this.checkOwner(token);
302
- return [...this.db.prepare('SELECT a.*,j.state,j.cancel_requested FROM attempts a JOIN jobs j ON j.id=a.job_id WHERE a.owner_incarnation=? AND a.fence=? ORDER BY a.created_at,a.id').all(token.ownerIncarnation, token.fence).map(a => ({
303
- ...token, attemptId: String(a.id), jobId: String(a.job_id), stage: String(a.stage), cleanup: a.cleanup, finished: a.finished_at !== null, jobState: String(a.state), cancelRequested: a.cancel_requested === 1,
304
- })), ...indexJobs.indexOwnerAttempts(this.db, token)];
305
- }
306
- /** Requires external custody proof for EVERY pending attempt, even after its job committed.
307
- * IDs are acknowledgements from the custody owner, never evidence inferred from PID/timeouts here. */
308
- recoverOwner(expected, provedAttemptIds) {
309
- if (!Array.isArray(provedAttemptIds) || new Set(provedAttemptIds).size !== provedAttemptIds.length)
310
- throw new StoreError('invalid-input');
311
- for (const id of provedAttemptIds)
312
- text(id, 256);
313
- this.transaction(() => {
314
- this.checkOwner(expected);
315
- const pending = this.ownerAttempts(expected).filter(a => a.cleanup === 'pending');
316
- const proof = new Set(provedAttemptIds);
317
- if (proof.size !== pending.length || pending.some(a => !proof.has(a.attemptId)))
318
- throw new StoreError('owner-busy');
319
- for (const attempt of pending) {
320
- const path = join(this.root, 'staging', attempt.attemptId);
321
- if (pathExists(path)) {
322
- if (attempt.kind === 'index')
323
- this.checkIndexStaging(attempt, path);
324
- else
325
- safeDirectory(path);
326
- rmSync(path, { recursive: true });
327
- syncDirectory(dirname(path));
328
- }
329
- if (attempt.kind === 'index') {
330
- indexJobs.recoverIndexAttempt(this.db, attempt);
331
- continue;
332
- }
333
- this.db.prepare('DELETE FROM staging_pins WHERE attempt_id=?').run(attempt.attemptId);
334
- this.db.prepare("UPDATE attempts SET cleanup='settled',stage='settled',finished_at=COALESCE(finished_at,?) WHERE id=?").run(now(), attempt.attemptId);
335
- if (!attempt.finished)
336
- this.db.prepare("UPDATE jobs SET state=CASE WHEN cancel_requested=1 THEN 'cancelled' ELSE 'failed' END,reason='unclassified',retryable=0,retry_after_ms=NULL WHERE id=? AND state IN ('running','cleanup-pending')").run(attempt.jobId);
337
- }
338
- this.db.prepare('UPDATE worker_owner SET incarnation=NULL,fence=fence+1 WHERE id=1').run();
339
- });
340
- }
341
- checkpointAttempt(token, stage) {
342
- const stages = ['queued', 'acquiring', 'extracting', 'indexing', 'promoting', 'cleanup', 'settled'];
343
- if (!stages.includes(stage) || stage === 'settled')
344
- throw new StoreError('invalid-input');
345
- this.transaction(() => {
346
- const attempt = this.checkAttempt(token);
347
- if (stages.indexOf(stage) < stages.indexOf(String(attempt.stage)) || !stages.includes(String(attempt.stage)))
348
- throw new StoreError('invalid-input');
349
- this.db.prepare('UPDATE attempts SET stage=? WHERE id=?').run(stage, token.attemptId);
350
- });
351
- }
352
- jobCancellationRequested(jobId) {
353
- text(jobId, 256);
354
- const index = indexJobs.indexJob(this.db, jobId);
355
- if (index)
356
- return index.cancel_requested === 1;
357
- const row = this.db.prepare('SELECT j.cancel_requested,i.deleted_at FROM jobs j JOIN items i ON i.id=j.item_id WHERE j.id=?').get(jobId);
358
- if (!row)
359
- throw new StoreError('unavailable');
360
- return row.cancel_requested === 1 || row.deleted_at !== null;
361
- }
362
- retryJob(jobId, options = {}) {
363
- if (indexJobs.indexJob(this.db, jobId)) {
364
- if (options.automatic || options.notBefore !== undefined)
365
- throw new StoreError('invalid-input');
366
- indexJobs.retryIndexJob(this.db, jobId);
367
- return;
368
- }
369
- text(jobId, 256);
370
- if (options.automatic !== undefined && typeof options.automatic !== 'boolean')
371
- throw new StoreError('invalid-input');
372
- if (options.notBefore !== undefined && (typeof options.notBefore !== 'string' || !Number.isFinite(Date.parse(options.notBefore))))
373
- throw new StoreError('invalid-input');
374
- this.transaction(() => {
375
- const job = this.db.prepare('SELECT j.*,i.deleted_at FROM jobs j JOIN items i ON i.id=j.item_id WHERE j.id=?').get(jobId);
376
- if (!job || job.deleted_at || job.cancel_requested || !['failed', 'needs-auth', 'needs-permission', 'unsupported'].includes(String(job.state)))
377
- throw new StoreError('unavailable');
378
- if (this.db.prepare("SELECT count(*) n FROM attempts WHERE job_id=? AND (cleanup='pending' OR finished_at IS NULL)").get(jobId)?.n)
379
- throw new StoreError('owner-busy');
380
- if (this.db.prepare("SELECT count(*) n FROM jobs WHERE item_id=? AND id<>? AND state IN ('queued','running','delayed','cleanup-pending')").get(String(job.item_id), jobId)?.n)
381
- throw new StoreError('owner-busy');
382
- let due = options.notBefore === undefined ? Date.now() : Date.parse(options.notBefore);
383
- if (options.automatic) {
384
- if (job.retryable !== 1 || !['unavailable', 'rate-limited'].includes(String(job.reason)) || Number(job.attempt_count) >= 3)
385
- throw new StoreError('unavailable');
386
- const backoff = Math.max(Number(job.retry_after_ms ?? 0), 1000 * 2 ** Math.max(0, Number(job.attempt_count) - 1));
387
- due = Math.max(due, Date.now() + backoff);
388
- }
389
- const notBefore = new Date(due).toISOString();
390
- this.db.prepare("UPDATE jobs SET state=?,not_before=?,reason=NULL,retryable=0,retry_after_ms=NULL WHERE id=?").run(due > Date.now() ? 'delayed' : 'queued', notBefore, jobId);
391
- });
392
- }
393
- checkOwner(token) {
394
- text(token.ownerIncarnation, 256);
395
- integer(token.fence);
396
- const owner = this.db.prepare('SELECT * FROM worker_owner WHERE id=1').get();
397
- if (owner?.incarnation !== token.ownerIncarnation || owner?.fence !== token.fence)
398
- throw new StoreError('owner-busy');
399
- }
400
- submitSource(input) {
401
- text(input.sourceKey);
402
- text(input.locator, 16384);
403
- text(input.idempotencyKey, 256);
404
- if (!['article', 'pdf', 'media'].includes(input.kind))
405
- throw new StoreError('invalid-input');
406
- const realm = input.realm ?? '';
407
- if (realm)
408
- text(realm, 256);
409
- if (input.intent !== undefined && !['save', 'refresh'].includes(input.intent))
410
- throw new StoreError('invalid-input');
411
- return this.transaction(() => {
412
- const prior = this.db.prepare('SELECT s.*,r.source_key,r.realm_key,i.kind,i.deleted_at FROM submissions s JOIN source_refs r ON r.id=s.source_id JOIN items i ON i.id=s.item_id WHERE idempotency_key=?').get(input.idempotencyKey);
413
- if (prior) {
414
- if (prior.deleted_at)
415
- throw new StoreError('unavailable');
416
- if (prior.kind !== input.kind)
417
- throw new StoreError('invalid-input');
418
- if (prior.source_key !== input.sourceKey || prior.realm_key !== realm || prior.intent !== (input.intent ?? 'save'))
419
- throw new StoreError('invalid-input');
420
- return { itemId: prior.item_id, jobId: prior.job_id, submissionId: prior.id, duplicate: true };
421
- }
422
- let source = this.db.prepare('SELECT r.*,i.deleted_at,i.kind FROM source_refs r JOIN items i ON i.id=r.item_id WHERE source_key=? AND realm_key=?').get(input.sourceKey, realm);
423
- if (source?.deleted_at)
424
- throw new StoreError('unavailable');
425
- if (source && source.kind !== input.kind)
426
- throw new StoreError('invalid-input');
427
- if (!source) {
428
- // Devices of one library agree on the id; outsiders cannot derive it from a URL.
429
- const library = String(this.db.prepare('SELECT uuid FROM library_info WHERE id=1').get()?.uuid);
430
- const itemId = digest(JSON.stringify(['item-v1', library, realm, input.sourceKey]));
431
- if (this.db.prepare('SELECT 1 FROM items WHERE id=?').get(itemId))
432
- throw new StoreError('integrity');
433
- foundation.mutate(this.db, 'item', itemId, 'created', { kind: input.kind, sourceKey: input.sourceKey, realm, locator: input.locator, createdAt: now() });
434
- foundation.syncUris(this.db, itemId);
435
- source = { id: foundation.sourceRefId(itemId), item_id: itemId };
436
- }
437
- const active = this.db.prepare("SELECT id FROM jobs WHERE item_id=? AND state IN ('queued','running','delayed','cleanup-pending') ORDER BY created_at LIMIT 1").get(source.item_id);
438
- const jobId = active ? String(active.id) : randomUUID();
439
- if (!active)
440
- this.db.prepare("INSERT INTO jobs(id,item_id,state,created_at) VALUES(?,?,'queued',?)").run(jobId, source.item_id, now());
441
- const submissionId = randomUUID();
442
- this.db.prepare('INSERT INTO submissions VALUES(?,?,?,?,?,?,?)').run(submissionId, input.idempotencyKey, source.id, source.item_id, jobId, now(), input.intent ?? 'save');
443
- return { itemId: source.item_id, jobId, submissionId, duplicate: false };
444
- });
445
- }
446
- listJobs() { return [...this.db.prepare('SELECT j.* FROM jobs j JOIN items i ON i.id=j.item_id WHERE i.deleted_at IS NULL ORDER BY j.created_at,j.id').all(), ...indexJobs.listIndexJobs(this.db)].sort((a, b) => String(a.created_at).localeCompare(String(b.created_at)) || String(a.id).localeCompare(String(b.id))); }
447
- sourceForJob(jobId) { text(jobId, 256); const row = this.db.prepare('SELECT i.id itemId,i.kind,r.source_key sourceKey,r.locator,r.realm_key realm FROM jobs j JOIN items i ON i.id=j.item_id JOIN source_refs r ON r.item_id=i.id WHERE j.id=? AND i.deleted_at IS NULL ORDER BY r.id LIMIT 1').get(jobId); return row ?? null; }
448
- claimNext(token) { this.checkOwner(token); const row = this.db.prepare("SELECT j.id FROM jobs j JOIN items i ON i.id=j.item_id WHERE j.state IN ('queued','delayed') AND j.cancel_requested=0 AND i.deleted_at IS NULL AND (j.not_before IS NULL OR j.not_before<=?) ORDER BY j.priority DESC,j.created_at,j.id LIMIT 1").get(now()); return row ? this.claimJob(token, String(row.id)) : null; }
449
- claimJob(token, jobId) {
450
- text(jobId, 256);
451
- return this.transaction(() => {
452
- this.checkOwner(token);
453
- if (this.db.prepare("SELECT count(*) AS n FROM index_attempts WHERE cleanup='pending'").get()?.n)
454
- throw new StoreError('owner-busy');
455
- const job = this.db.prepare('SELECT j.*,i.deleted_at FROM jobs j JOIN items i ON i.id=j.item_id WHERE j.id=?').get(jobId);
456
- if (!job || !['queued', 'delayed'].includes(String(job.state)) || job.cancel_requested || job.deleted_at || (job.not_before && String(job.not_before) > now()))
457
- throw new StoreError('unavailable');
458
- const attemptId = randomUUID();
459
- this.db.prepare("UPDATE jobs SET state='running',attempt_count=attempt_count+1 WHERE id=?").run(jobId);
460
- this.db.prepare("INSERT INTO attempts(id,job_id,owner_incarnation,fence,stage,cleanup,created_at) VALUES(?,?,?,?,'acquiring','pending',?)").run(attemptId, jobId, token.ownerIncarnation, token.fence, now());
461
- return { ...token, jobId, attemptId };
462
- });
463
- }
464
- checkAttempt(token, itemId) {
465
- this.checkOwner(token);
466
- text(token.attemptId, 256);
467
- text(token.jobId, 256);
468
- const row = this.db.prepare('SELECT a.*,j.item_id,j.state,j.cancel_requested,i.deleted_at FROM attempts a JOIN jobs j ON j.id=a.job_id JOIN items i ON i.id=j.item_id WHERE a.id=? AND a.job_id=?').get(token.attemptId, token.jobId);
469
- if (!row || row.owner_incarnation !== token.ownerIncarnation || row.fence !== token.fence || row.finished_at || row.state !== 'running')
470
- throw new StoreError('owner-busy');
471
- if (row.cancel_requested || row.deleted_at)
472
- throw new StoreError('cancelled');
473
- if (itemId && row.item_id !== itemId)
474
- throw new StoreError('invalid-input');
475
- return row;
476
- }
477
- requestCancellation(jobId) {
478
- text(jobId, 256);
479
- if (indexJobs.indexJob(this.db, jobId)) {
480
- indexJobs.cancelIndexJob(this.db, jobId);
481
- return;
482
- }
483
- this.transaction(() => {
484
- const job = this.db.prepare('SELECT state FROM jobs WHERE id=?').get(jobId);
485
- if (!job)
486
- throw new StoreError('unavailable');
487
- if (['succeeded', 'cancelled'].includes(String(job.state)))
488
- return;
489
- this.db.prepare("UPDATE jobs SET cancel_requested=1,state=CASE WHEN state IN ('queued','delayed','needs-auth','needs-permission','unsupported','failed') THEN 'cancelled' ELSE state END WHERE id=?").run(jobId);
490
- });
491
- }
492
- failAttempt(token, reason = 'unclassified', options = {}) {
493
- if (options.retryable !== undefined && typeof options.retryable !== 'boolean')
494
- throw new StoreError('invalid-input');
495
- if (options.retryAfterMs !== undefined)
496
- integer(options.retryAfterMs, 86400000);
497
- if (!/^[a-z][a-z-]{0,63}$/.test(reason))
498
- throw new StoreError('invalid-input');
499
- this.transaction(() => {
500
- this.checkOwner(token);
501
- const a = this.db.prepare('SELECT a.*,j.cancel_requested FROM attempts a JOIN jobs j ON j.id=a.job_id WHERE a.id=? AND a.job_id=?').get(token.attemptId, token.jobId);
502
- if (!a || a.owner_incarnation !== token.ownerIncarnation || a.fence !== token.fence || a.finished_at)
503
- throw new StoreError('owner-busy');
504
- this.db.prepare("UPDATE jobs SET state=?,reason=?,retryable=?,retry_after_ms=? WHERE id=?").run(a.cancel_requested ? 'cancelled' : reason === 'repair-auth' ? 'needs-auth' : reason === 'grant-permission' ? 'needs-permission' : reason === 'unsupported' ? 'unsupported' : 'failed', reason, Number(options.retryable === true), options.retryAfterMs ?? null, token.jobId);
505
- this.db.prepare("UPDATE attempts SET stage='cleanup',finished_at=? WHERE id=?").run(now(), token.attemptId);
506
- });
507
- }
508
- /** Caller must first settle owned external children. This only releases storage staging pins. */
509
- settleAttempt(token) {
510
- this.transaction(() => {
511
- this.checkOwner(token);
512
- const a = this.db.prepare('SELECT * FROM attempts WHERE id=? AND job_id=?').get(token.attemptId, token.jobId);
513
- if (!a || a.owner_incarnation !== token.ownerIncarnation || a.fence !== token.fence || !a.finished_at)
514
- throw new StoreError('owner-busy');
515
- const path = join(this.root, 'staging', token.attemptId);
516
- if (pathExists(path)) {
517
- safeDirectory(path);
518
- rmSync(path, { recursive: true });
519
- syncDirectory(dirname(path));
520
- }
521
- this.db.prepare('DELETE FROM staging_pins WHERE attempt_id=?').run(token.attemptId);
522
- this.db.prepare("UPDATE attempts SET cleanup='settled',stage='settled' WHERE id=?").run(token.attemptId);
523
- });
524
- }
525
- async putBlob(token, expectedHash, expectedBytes, source) {
526
- hash(expectedHash);
527
- integer(expectedBytes, limits.pdfBytes);
528
- const relativePath = `blobs/sha256/${expectedHash.slice(0, 2)}/${expectedHash}`;
529
- this.transaction(() => {
530
- this.checkAttempt(token);
531
- const pins = this.db.prepare("SELECT COALESCE(sum(bytes),0) n FROM staging_pins").get();
532
- if (Number(pins.n) + expectedBytes > limits.stagingBytes)
533
- throw new StoreError('limit-exceeded');
534
- const free = statfsSync(this.root);
535
- if (free.bavail * free.bsize < limits.freeReserveBytes + expectedBytes)
536
- throw new StoreError('limit-exceeded');
537
- const existing = this.db.prepare('SELECT bytes FROM staging_pins WHERE attempt_id=? AND sha256=?').get(token.attemptId, expectedHash);
538
- if (existing && existing.bytes !== expectedBytes)
539
- throw new StoreError('integrity');
540
- this.db.prepare("INSERT OR IGNORE INTO staging_pins VALUES(?,?,?,'staging')").run(token.attemptId, expectedHash, expectedBytes);
541
- });
542
- const stage = safeDirectory(join(this.root, 'staging', token.attemptId), true), temp = join(stage, randomUUID() + '.tmp');
543
- const fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
544
- let count = 0;
545
- const h = createHash('sha256');
546
- try {
547
- for await (const chunk of source) {
548
- if (!(chunk instanceof Uint8Array))
549
- throw new StoreError('invalid-input');
550
- count += chunk.byteLength;
551
- if (count > expectedBytes)
552
- throw new StoreError('limit-exceeded');
553
- h.update(chunk);
554
- let offset = 0;
555
- while (offset < chunk.byteLength)
556
- offset += writeSync(fd, chunk, offset, chunk.byteLength - offset);
557
- }
558
- if (count !== expectedBytes || h.digest('hex') !== expectedHash)
559
- throw new StoreError('integrity');
560
- fsyncSync(fd);
561
- }
562
- catch (error) {
563
- closeSync(fd);
564
- unlinkSync(temp);
565
- throw error;
566
- }
567
- closeSync(fd);
568
- syncDirectory(stage);
569
- this.options.fault?.('staged');
570
- const target = join(this.root, relativePath);
571
- safeDirectory(dirname(target), true);
572
- // Atomic create-if-absent publication. A hard-link plus unlink avoids rename overwriting immutable bytes.
573
- this.transaction(() => {
574
- this.checkAttempt(token);
575
- try {
576
- linkSync(temp, target);
577
- }
578
- catch (error) {
579
- if (error.code !== 'EEXIST')
580
- throw error;
581
- verifyFile(target, expectedHash, expectedBytes);
582
- }
583
- unlinkSync(temp);
584
- syncDirectory(dirname(target));
585
- syncDirectory(stage);
586
- verifyFile(target, expectedHash, expectedBytes);
587
- this.db.prepare("UPDATE staging_pins SET state='published' WHERE attempt_id=? AND sha256=?").run(token.attemptId, expectedHash);
588
- });
589
- this.options.fault?.('published');
590
- return { sha256: expectedHash, bytes: expectedBytes, relativePath };
591
- }
592
- commitRevision(input) {
593
- hash(input.bundleHash);
594
- if (typeof input.markdown !== 'string' || Buffer.byteLength(input.markdown) > limits.markdownBytes)
595
- throw new StoreError('limit-exceeded');
596
- text(input.itemId, 256);
597
- text(input.extractor.name, 256);
598
- text(input.extractor.version, 128);
599
- if (input.title)
600
- text(input.title, 4096);
601
- if (!['complete', 'partial'].includes(input.status) || input.assets.length > limits.imagesPerItem + 1)
602
- throw new StoreError('invalid-input');
603
- const metadata = input.metadata ?? [];
604
- if (metadata.length > limits.metadataClaims || Buffer.byteLength(JSON.stringify(metadata)) > limits.metadataBytes)
605
- throw new StoreError('limit-exceeded');
606
- for (const claim of metadata) {
607
- try {
608
- validateMetadataClaim(claim);
609
- }
610
- catch {
611
- throw new StoreError('invalid-input');
612
- }
613
- }
614
- const metadataWarnings = input.metadataWarnings ?? [];
615
- try {
616
- validateMetadataWarnings(metadataWarnings);
617
- }
618
- catch {
619
- throw new StoreError('invalid-input');
620
- }
621
- const imported = input.importedState;
622
- if (imported) {
623
- text(imported.namespace, 256);
624
- text(imported.sourceKey, 8192);
625
- hash(imported.sourceHash);
626
- text(imported.importerVersion, 128);
627
- if (!['private', 'published'].includes(imported.privacy) || !Array.isArray(imported.notes) || imported.notes.length > 1000 || Buffer.byteLength(JSON.stringify(imported)) > 4 * 1024 ** 2)
628
- throw new StoreError('limit-exceeded');
629
- const noteIds = new Set();
630
- for (const note of imported.notes) {
631
- text(note.id, 256);
632
- text(note.body, 65536);
633
- if (noteIds.has(note.id))
634
- throw new StoreError('invalid-input');
635
- noteIds.add(note.id);
636
- }
637
- if (imported.readingState !== undefined && (!imported.readingState || Array.isArray(imported.readingState) || typeof imported.readingState !== 'object'))
638
- throw new StoreError('invalid-input');
639
- }
640
- for (const asset of input.assets) {
641
- hash(asset.sha256);
642
- const transcript = asset.mediaType === 'application/json' && asset.relationship === 'transcript';
643
- integer(asset.bytes, transcript ? 4 * 1024 ** 2 : asset.mediaType === 'application/pdf' ? limits.pdfBytes : limits.imageBytes);
644
- if (!/^image\/[a-z0-9.+-]+$/.test(asset.mediaType) && asset.mediaType !== 'application/pdf' && !transcript)
645
- throw new StoreError('invalid-input');
646
- if (asset.relativePath !== `blobs/sha256/${asset.sha256.slice(0, 2)}/${asset.sha256}`)
647
- throw new StoreError('integrity');
648
- verifyFile(confinedFile(this.root, join(this.root, asset.relativePath)), asset.sha256, asset.bytes);
649
- }
650
- const payloadHash = digest(JSON.stringify({ markdown: input.markdown, title: input.title ?? '', extractor: input.extractor, status: input.status, assets: input.assets, metadata, warnings: input.warnings ?? [], ...(metadataWarnings.length ? { metadataWarnings } : {}) }));
651
- const result = this.transaction(() => {
652
- this.checkAttempt(input, input.itemId);
653
- for (const asset of input.assets) {
654
- const pin = this.db.prepare("SELECT bytes FROM staging_pins WHERE attempt_id=? AND sha256=? AND state='published'").get(input.attemptId, asset.sha256);
655
- if (!pin || pin.bytes !== asset.bytes)
656
- throw new StoreError('integrity');
657
- }
658
- const existing = this.db.prepare('SELECT id,payload_hash FROM revisions WHERE item_id=? AND bundle_hash=?').get(input.itemId, input.bundleHash);
659
- const item = this.db.prepare('SELECT current_revision,title FROM items WHERE id=?').get(input.itemId);
660
- // Content-addressed: the same capture on two devices is the same revision object.
661
- const revisionId = existing ? String(existing.id) : digest(JSON.stringify(['revision-v1', input.itemId, input.bundleHash]));
662
- if (!existing) {
663
- this.db.prepare('INSERT INTO revisions(id,item_id,parent_id,bundle_hash,extractor_name,extractor_version,captured_at,status,warnings_json,payload_hash,metadata_warnings_json) VALUES(?,?,?,?,?,?,?,?,?,?,?)').run(revisionId, input.itemId, item.current_revision ?? null, input.bundleHash, input.extractor.name, input.extractor.version, now(), input.status, JSON.stringify(input.warnings ?? []), payloadHash, JSON.stringify(metadataWarnings));
664
- for (const [ordinal, asset] of input.assets.entries()) {
665
- const existingAsset = this.db.prepare('SELECT bytes FROM assets WHERE sha256=?').get(asset.sha256);
666
- if (existingAsset && existingAsset.bytes !== asset.bytes)
667
- throw new StoreError('integrity');
668
- this.db.prepare('INSERT OR IGNORE INTO assets VALUES(?,?,?,?)').run(asset.sha256, asset.bytes, asset.mediaType, asset.relativePath);
669
- this.db.prepare('INSERT INTO revision_assets VALUES(?,?,?,?,?,?,?)').run(revisionId, asset.sha256, ordinal, asset.relationship ?? 'image', null, asset.alt ?? null, asset.caption ?? null);
670
- }
671
- this.db.prepare('INSERT INTO representations(id,revision_id,kind,mime_type,sha256,bytes,markdown,extractor_version) VALUES(?,?,?,?,?,?,?,?)').run(randomUUID(), revisionId, 'markdown', 'text/markdown', digest(input.markdown), Buffer.byteLength(input.markdown), input.markdown, input.extractor.version);
672
- for (const claim of metadata)
673
- this.db.prepare('INSERT INTO metadata_claims(id,revision_id,field,value_json,source,observed_at,rule,timezone_known,observed_json,locator,selected,extraction_version,date_precision) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)').run(randomUUID(), revisionId, claim.field, JSON.stringify(claim.value), claim.source, claim.observedAt, claim.rule ?? null, claim.timezoneKnown === undefined ? null : Number(claim.timezoneKnown), claim.observed === undefined ? null : JSON.stringify(claim.observed), claim.locator ?? null, claim.selected === undefined ? null : Number(claim.selected), claim.extractionVersion ?? null, claim.datePrecision ?? null);
674
- this.db.prepare('INSERT INTO blocks(id,revision_id,ordinal,kind,text) VALUES(?,?,0,?,?)').run(randomUUID(), revisionId, 'document', input.markdown);
675
- }
676
- else {
677
- if (existing.payload_hash !== payloadHash)
678
- throw new StoreError('integrity');
679
- const prior = this.db.prepare("SELECT sha256 FROM representations WHERE revision_id=? AND kind='markdown'").get(revisionId);
680
- if (prior?.sha256 !== digest(input.markdown))
681
- throw new StoreError('integrity');
682
- }
683
- if (item.current_revision !== revisionId) {
684
- foundation.mutate(this.db, 'item', input.itemId, 'currentRevision', revisionId);
685
- if (item.title !== (input.title ?? ''))
686
- foundation.mutate(this.db, 'item', input.itemId, 'title', input.title ?? '');
687
- const stored = this.db.prepare("SELECT markdown FROM representations WHERE revision_id=? AND kind='markdown'").get(revisionId);
688
- indexReadable(this.db, input.itemId, revisionId, input.title ?? '', String(stored.markdown));
689
- foundation.syncUris(this.db, input.itemId);
690
- this.bumpEpoch();
691
- }
692
- if (imported) {
693
- const receipt = this.lookupImportReceipt(imported.namespace, imported.sourceKey, imported.sourceHash);
694
- if (receipt && receipt.itemId !== input.itemId)
695
- throw new StoreError('integrity');
696
- for (const note of imported.notes)
697
- foundation.importNote(this.db, { itemId: input.itemId, revisionId, namespace: imported.namespace, sourceKey: imported.sourceKey, noteId: note.id, body: note.body });
698
- const state = JSON.stringify({ ...imported.readingState, sourcePrivacy: imported.privacy });
699
- const prior = this.db.prepare('SELECT state_json FROM reading_state WHERE item_id=?').get(input.itemId);
700
- // Existing reading choices remain authoritative; import never overwrites them.
701
- if (!prior)
702
- foundation.setReadingState(this.db, input.itemId, JSON.parse(state));
703
- else {
704
- const existing = JSON.parse(String(prior.state_json));
705
- const sourcePrivacy = existing.sourcePrivacy === 'published' && imported.privacy === 'published' ? 'published' : 'private';
706
- if (sourcePrivacy !== existing.sourcePrivacy)
707
- foundation.setReadingState(this.db, input.itemId, { ...existing, sourcePrivacy });
708
- }
709
- this.db.prepare('INSERT OR IGNORE INTO import_receipts VALUES(?,?,?,?,?,?)').run(imported.namespace, imported.sourceKey, imported.sourceHash, imported.importerVersion, input.itemId, 'imported');
710
- }
711
- this.db.prepare("UPDATE jobs SET state='succeeded',reason=NULL WHERE id=?").run(input.jobId);
712
- this.db.prepare("UPDATE attempts SET stage='cleanup',finished_at=? WHERE id=?").run(now(), input.attemptId);
713
- this.options.fault?.('transaction');
714
- return { revisionId, duplicate: !!existing };
715
- });
716
- this.options.fault?.('committed');
717
- return result;
718
- }
719
- list() { return this.db.prepare('SELECT id itemId,kind,title,current_revision revisionId FROM items WHERE deleted_at IS NULL ORDER BY created_at,id').all(); }
720
- read(itemId) {
721
- text(itemId, 256);
722
- const item = this.db.prepare("SELECT i.id itemId,i.kind,i.title,r.id revisionId,r.status,p.markdown FROM items i JOIN revisions r ON r.id=i.current_revision JOIN representations p ON p.revision_id=r.id AND p.kind='markdown' WHERE i.id=? AND i.deleted_at IS NULL").get(itemId);
723
- if (!item)
724
- return null;
725
- const assets = this.db.prepare('SELECT a.*,r.relationship,r.alt,r.caption FROM revision_assets r JOIN assets a ON a.sha256=r.sha256 WHERE r.revision_id=? ORDER BY r.ordinal').all(item.revisionId).map(a => ({ sha256: String(a.sha256), bytes: Number(a.bytes), relativePath: String(a.relative_path), mediaType: String(a.media_type), relationship: String(a.relationship), ...(a.alt ? { alt: String(a.alt) } : {}), ...(a.caption ? { caption: String(a.caption) } : {}) }));
726
- const metadata = this.db.prepare('SELECT * FROM metadata_claims WHERE revision_id=? ORDER BY rowid').all(item.revisionId).map(c => ({ field: String(c.field), value: JSON.parse(String(c.value_json)), source: String(c.source), observedAt: String(c.observed_at), ...(c.rule ? { rule: String(c.rule) } : {}), ...(c.timezone_known === null ? {} : { timezoneKnown: !!c.timezone_known }), ...(c.observed_json === null ? {} : { observed: JSON.parse(String(c.observed_json)) }), ...(c.locator === null ? {} : { locator: String(c.locator) }), ...(c.selected === null ? {} : { selected: !!c.selected }), ...(c.extraction_version === null ? {} : { extractionVersion: String(c.extraction_version) }), ...(c.date_precision === null ? {} : { datePrecision: String(c.date_precision) }) }));
727
- const metadataWarnings = JSON.parse(String(this.db.prepare('SELECT metadata_warnings_json FROM revisions WHERE id=?').get(item.revisionId)?.metadata_warnings_json));
728
- validateMetadataWarnings(metadataWarnings);
729
- return { ...item, assets, metadata, metadataWarnings };
730
- }
731
- readBlob(reference) {
732
- hash(reference.sha256);
733
- integer(reference.bytes, limits.pdfBytes);
734
- if (reference.relativePath !== `blobs/sha256/${reference.sha256.slice(0, 2)}/${reference.sha256}`)
735
- throw new StoreError('integrity');
736
- const path = confinedFile(this.root, join(this.root, reference.relativePath));
737
- return readVerifiedFile(path, reference.sha256, reference.bytes);
738
- }
739
- search(query, count = 20) { text(query, 4096); integer(count, 100); return lexicalSearch(this.db, query, count).map(({ itemId, title, revisionId }) => ({ itemId, title, revisionId })); }
740
- delete(itemId) {
741
- text(itemId, 256);
742
- this.transaction(() => {
743
- const row = this.db.prepare('SELECT 1 FROM items WHERE id=? AND deleted_at IS NULL').get(itemId);
744
- if (row) {
745
- foundation.mutate(this.db, 'item', itemId, 'deleted', now());
746
- unindexReadable(this.db, itemId);
747
- this.db.prepare("UPDATE jobs SET cancel_requested=1,state=CASE WHEN state IN ('queued','delayed','needs-auth','needs-permission','unsupported','failed') THEN 'cancelled' ELSE state END WHERE item_id=? AND state NOT IN ('succeeded','failed','cancelled')").run(itemId);
748
- this.bumpEpoch();
749
- }
750
- });
751
- }
752
- /** Online DB snapshot only; P12 pairs this with blobs while the returned fence remains active. */
753
- async createBackupSnapshot() {
754
- const id = randomUUID();
755
- const epoch = this.transaction(() => { const e = this.revisionEpoch; this.db.prepare("INSERT INTO backup_fences VALUES(?,?,'active',?)").run(id, e, now()); return e; });
756
- const path = join(this.root, 'backups', id + '.sqlite');
757
- try {
758
- await backup(this.db, path);
759
- chmodSync(path, 0o600);
760
- const fd = openSync(path, constants.O_RDONLY);
761
- try {
762
- fsyncSync(fd);
763
- }
764
- finally {
765
- closeSync(fd);
766
- }
767
- syncDirectory(dirname(path));
768
- return { id, path, revisionEpoch: epoch };
769
- }
770
- catch (error) {
771
- this.db.prepare('DELETE FROM backup_fences WHERE id=?').run(id);
772
- throw error;
773
- }
774
- }
775
- finishBackup(id) { text(id, 256); this.db.prepare("UPDATE backup_fences SET state='complete' WHERE id=?").run(id); }
776
- /** Hold the SQLite writer fence across synchronous GC inspection and removal. */
777
- withGcFence(expectedEpoch, callback) {
778
- integer(expectedEpoch);
779
- return this.transaction(() => {
780
- if (this.revisionEpoch !== expectedEpoch || this.ownerSnapshot() !== null || this.pendingCleanupCount() > 0 || this.hasGcFence())
781
- throw new StoreError('owner-busy');
782
- const references = this.db.prepare('SELECT sha256,bytes,relative_path relativePath FROM assets ORDER BY sha256').all();
783
- return callback(references);
784
- });
785
- }
786
- /** Runtime GC pins: recorded model artifacts and embedding helpers of active or building generations. */
787
- runtimeReferences() {
788
- const modelArtifacts = this.db.prepare('SELECT id,manifest_json FROM model_artifacts ORDER BY id').all().map(r => ({ id: String(r.id), manifest: String(r.manifest_json) }));
789
- const helpers = new Set();
790
- let unmapped = 0;
791
- for (const row of this.db.prepare("SELECT DISTINCT s.identity_json FROM embedding_spaces s JOIN index_generations g ON g.fingerprint=s.fingerprint WHERE g.state='building' OR g.id IN (SELECT generation_id FROM active_indexes)").all()) {
792
- let identity;
793
- try {
794
- identity = JSON.parse(String(row.identity_json));
795
- }
796
- catch {
797
- throw new StoreError('integrity');
798
- }
799
- const sha = identity?.helperSha256;
800
- if (typeof sha === 'string' && /^[a-f0-9]{64}$/.test(sha))
801
- helpers.add(sha);
802
- else
803
- unmapped++;
804
- }
805
- return { modelArtifacts, embeddingHelpers: [...helpers].sort(), unmappedSpaces: unmapped };
806
- }
807
- /** GC must honor both durable staging pins and active backup fences. */
808
- pendingCleanupCount() { return Number(this.db.prepare("SELECT (SELECT count(*) FROM attempts WHERE cleanup='pending')+(SELECT count(*) FROM index_attempts WHERE cleanup='pending') AS n").get()?.n); }
809
- hasGcFence() { return Number(this.db.prepare("SELECT (SELECT count(*) FROM staging_pins)+(SELECT count(*) FROM backup_fences WHERE state='active')+(SELECT count(*) FROM index_attempts WHERE cleanup='pending')+(SELECT count(*) FROM index_jobs WHERE state IN ('running','cleanup-pending')) n").get()?.n) > 0; }
810
- }