@hatk/hatk 0.0.1-alpha.9 → 0.0.1-alpha.92

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/dist/adapter.d.ts +19 -0
  2. package/dist/adapter.d.ts.map +1 -0
  3. package/dist/adapter.js +113 -0
  4. package/dist/backfill.d.ts +29 -2
  5. package/dist/backfill.d.ts.map +1 -1
  6. package/dist/backfill.js +142 -51
  7. package/dist/car.d.ts +20 -0
  8. package/dist/car.d.ts.map +1 -1
  9. package/dist/car.js +108 -0
  10. package/dist/cli.js +247 -1043
  11. package/dist/config.d.ts +145 -1
  12. package/dist/config.d.ts.map +1 -1
  13. package/dist/config.js +61 -8
  14. package/dist/database/adapter-factory.d.ts +6 -0
  15. package/dist/database/adapter-factory.d.ts.map +1 -0
  16. package/dist/database/adapter-factory.js +20 -0
  17. package/dist/database/adapters/duckdb-search.d.ts +12 -0
  18. package/dist/database/adapters/duckdb-search.d.ts.map +1 -0
  19. package/dist/database/adapters/duckdb-search.js +27 -0
  20. package/dist/database/adapters/duckdb.d.ts +25 -0
  21. package/dist/database/adapters/duckdb.d.ts.map +1 -0
  22. package/dist/database/adapters/duckdb.js +161 -0
  23. package/dist/database/adapters/sqlite-search.d.ts +16 -0
  24. package/dist/database/adapters/sqlite-search.d.ts.map +1 -0
  25. package/dist/database/adapters/sqlite-search.js +76 -0
  26. package/dist/database/adapters/sqlite.d.ts +18 -0
  27. package/dist/database/adapters/sqlite.d.ts.map +1 -0
  28. package/dist/database/adapters/sqlite.js +88 -0
  29. package/dist/{db.d.ts → database/db.d.ts} +68 -6
  30. package/dist/database/db.d.ts.map +1 -0
  31. package/dist/database/db.js +1775 -0
  32. package/dist/database/dialect.d.ts +45 -0
  33. package/dist/database/dialect.d.ts.map +1 -0
  34. package/dist/database/dialect.js +72 -0
  35. package/dist/{fts.d.ts → database/fts.d.ts} +7 -0
  36. package/dist/database/fts.d.ts.map +1 -0
  37. package/dist/{fts.js → database/fts.js} +122 -34
  38. package/dist/database/index.d.ts +7 -0
  39. package/dist/database/index.d.ts.map +1 -0
  40. package/dist/database/index.js +6 -0
  41. package/dist/database/ports.d.ts +50 -0
  42. package/dist/database/ports.d.ts.map +1 -0
  43. package/dist/database/ports.js +1 -0
  44. package/dist/{schema.d.ts → database/schema.d.ts} +16 -3
  45. package/dist/database/schema.d.ts.map +1 -0
  46. package/dist/{schema.js → database/schema.js} +126 -44
  47. package/dist/dev-entry.d.ts +8 -0
  48. package/dist/dev-entry.d.ts.map +1 -0
  49. package/dist/dev-entry.js +127 -0
  50. package/dist/feeds.d.ts +26 -8
  51. package/dist/feeds.d.ts.map +1 -1
  52. package/dist/feeds.js +72 -16
  53. package/dist/hooks.d.ts +102 -0
  54. package/dist/hooks.d.ts.map +1 -0
  55. package/dist/hooks.js +186 -0
  56. package/dist/hydrate.d.ts +49 -6
  57. package/dist/hydrate.d.ts.map +1 -1
  58. package/dist/hydrate.js +21 -20
  59. package/dist/indexer.d.ts +167 -3
  60. package/dist/indexer.d.ts.map +1 -1
  61. package/dist/indexer.js +556 -94
  62. package/dist/jetstream.d.ts +118 -0
  63. package/dist/jetstream.d.ts.map +1 -0
  64. package/dist/jetstream.js +225 -0
  65. package/dist/labels.d.ts +36 -0
  66. package/dist/labels.d.ts.map +1 -1
  67. package/dist/labels.js +71 -6
  68. package/dist/lex-types.d.ts +2 -0
  69. package/dist/lex-types.d.ts.map +1 -1
  70. package/dist/lexicon-resolve.d.ts.map +1 -1
  71. package/dist/lexicon-resolve.js +27 -112
  72. package/dist/lexicons/com/atproto/label/defs.json +75 -0
  73. package/dist/lexicons/com/atproto/moderation/defs.json +30 -0
  74. package/dist/lexicons/com/atproto/repo/strongRef.json +21 -0
  75. package/dist/lexicons/dev/hatk/applyWrites.json +87 -0
  76. package/dist/lexicons/dev/hatk/createRecord.json +40 -0
  77. package/dist/lexicons/dev/hatk/createReport.json +48 -0
  78. package/dist/lexicons/dev/hatk/deleteRecord.json +25 -0
  79. package/dist/lexicons/dev/hatk/describeCollections.json +41 -0
  80. package/dist/lexicons/dev/hatk/describeFeeds.json +29 -0
  81. package/dist/lexicons/dev/hatk/describeLabels.json +45 -0
  82. package/dist/lexicons/dev/hatk/getFeed.json +30 -0
  83. package/dist/lexicons/dev/hatk/getPreferences.json +19 -0
  84. package/dist/lexicons/dev/hatk/getRecord.json +26 -0
  85. package/dist/lexicons/dev/hatk/getRecords.json +32 -0
  86. package/dist/lexicons/dev/hatk/putPreference.json +28 -0
  87. package/dist/lexicons/dev/hatk/putRecord.json +41 -0
  88. package/dist/lexicons/dev/hatk/searchRecords.json +32 -0
  89. package/dist/lexicons/dev/hatk/uploadBlob.json +23 -0
  90. package/dist/logger.d.ts +29 -0
  91. package/dist/logger.d.ts.map +1 -1
  92. package/dist/logger.js +29 -0
  93. package/dist/main.js +198 -75
  94. package/dist/mst.d.ts +15 -0
  95. package/dist/mst.d.ts.map +1 -1
  96. package/dist/mst.js +13 -0
  97. package/dist/oauth/db.d.ts +16 -1
  98. package/dist/oauth/db.d.ts.map +1 -1
  99. package/dist/oauth/db.js +74 -19
  100. package/dist/oauth/discovery.d.ts +10 -0
  101. package/dist/oauth/discovery.d.ts.map +1 -1
  102. package/dist/oauth/discovery.js +54 -2
  103. package/dist/oauth/server.d.ts +93 -1
  104. package/dist/oauth/server.d.ts.map +1 -1
  105. package/dist/oauth/server.js +416 -39
  106. package/dist/oauth/service-describe.d.ts +42 -0
  107. package/dist/oauth/service-describe.d.ts.map +1 -0
  108. package/dist/oauth/service-describe.js +108 -0
  109. package/dist/oauth/session.d.ts +32 -0
  110. package/dist/oauth/session.d.ts.map +1 -0
  111. package/dist/oauth/session.js +131 -0
  112. package/dist/opengraph.d.ts +10 -0
  113. package/dist/opengraph.d.ts.map +1 -1
  114. package/dist/opengraph.js +94 -42
  115. package/dist/pds-proxy.d.ts +82 -0
  116. package/dist/pds-proxy.d.ts.map +1 -0
  117. package/dist/pds-proxy.js +356 -0
  118. package/dist/private-collections.d.ts +13 -0
  119. package/dist/private-collections.d.ts.map +1 -0
  120. package/dist/private-collections.js +27 -0
  121. package/dist/push.d.ts +54 -0
  122. package/dist/push.d.ts.map +1 -0
  123. package/dist/push.js +367 -0
  124. package/dist/renderer.d.ts +27 -0
  125. package/dist/renderer.d.ts.map +1 -0
  126. package/dist/renderer.js +46 -0
  127. package/dist/resolve-hatk.d.ts +6 -0
  128. package/dist/resolve-hatk.d.ts.map +1 -0
  129. package/dist/resolve-hatk.js +20 -0
  130. package/dist/response.d.ts +16 -0
  131. package/dist/response.d.ts.map +1 -0
  132. package/dist/response.js +69 -0
  133. package/dist/scanner.d.ts +21 -0
  134. package/dist/scanner.d.ts.map +1 -0
  135. package/dist/scanner.js +88 -0
  136. package/dist/seed.d.ts +19 -0
  137. package/dist/seed.d.ts.map +1 -1
  138. package/dist/seed.js +43 -4
  139. package/dist/server-init.d.ts +8 -0
  140. package/dist/server-init.d.ts.map +1 -0
  141. package/dist/server-init.js +62 -0
  142. package/dist/server.d.ts +34 -3
  143. package/dist/server.d.ts.map +1 -1
  144. package/dist/server.js +1032 -649
  145. package/dist/setup.d.ts +28 -1
  146. package/dist/setup.d.ts.map +1 -1
  147. package/dist/setup.js +50 -3
  148. package/dist/spaces/blob.d.ts +51 -0
  149. package/dist/spaces/blob.d.ts.map +1 -0
  150. package/dist/spaces/blob.js +111 -0
  151. package/dist/spaces/credential.d.ts +62 -0
  152. package/dist/spaces/credential.d.ts.map +1 -0
  153. package/dist/spaces/credential.js +184 -0
  154. package/dist/spaces/engine.d.ts +97 -0
  155. package/dist/spaces/engine.d.ts.map +1 -0
  156. package/dist/spaces/engine.js +559 -0
  157. package/dist/spaces/guard.d.ts +41 -0
  158. package/dist/spaces/guard.d.ts.map +1 -0
  159. package/dist/spaces/guard.js +79 -0
  160. package/dist/spaces/identity.d.ts +37 -0
  161. package/dist/spaces/identity.d.ts.map +1 -0
  162. package/dist/spaces/identity.js +88 -0
  163. package/dist/spaces/index.d.ts +55 -0
  164. package/dist/spaces/index.d.ts.map +1 -0
  165. package/dist/spaces/index.js +133 -0
  166. package/dist/spaces/notify.d.ts +51 -0
  167. package/dist/spaces/notify.d.ts.map +1 -0
  168. package/dist/spaces/notify.js +121 -0
  169. package/dist/spaces/records.d.ts +37 -0
  170. package/dist/spaces/records.d.ts.map +1 -0
  171. package/dist/spaces/records.js +63 -0
  172. package/dist/spaces/store.d.ts +51 -0
  173. package/dist/spaces/store.d.ts.map +1 -0
  174. package/dist/spaces/store.js +99 -0
  175. package/dist/spaces/uri.d.ts +60 -0
  176. package/dist/spaces/uri.d.ts.map +1 -0
  177. package/dist/spaces/uri.js +97 -0
  178. package/dist/spaces/verify.d.ts +39 -0
  179. package/dist/spaces/verify.d.ts.map +1 -0
  180. package/dist/spaces/verify.js +87 -0
  181. package/dist/spaces/viewer.d.ts +51 -0
  182. package/dist/spaces/viewer.d.ts.map +1 -0
  183. package/dist/spaces/viewer.js +167 -0
  184. package/dist/spaces/visibility.d.ts +57 -0
  185. package/dist/spaces/visibility.d.ts.map +1 -0
  186. package/dist/spaces/visibility.js +81 -0
  187. package/dist/templates/feed.tpl +18 -0
  188. package/dist/templates/hook.tpl +5 -0
  189. package/dist/templates/label.tpl +15 -0
  190. package/dist/templates/og.tpl +17 -0
  191. package/dist/templates/seed.tpl +11 -0
  192. package/dist/templates/setup.tpl +5 -0
  193. package/dist/templates/test-feed.tpl +19 -0
  194. package/dist/templates/test-xrpc.tpl +19 -0
  195. package/dist/templates/xrpc.tpl +53 -0
  196. package/dist/test.d.ts +3 -1
  197. package/dist/test.d.ts.map +1 -1
  198. package/dist/test.js +60 -36
  199. package/dist/views.js +1 -1
  200. package/dist/vite-plugin.d.ts +1 -1
  201. package/dist/vite-plugin.d.ts.map +1 -1
  202. package/dist/vite-plugin.js +254 -66
  203. package/dist/xrpc.d.ts +101 -12
  204. package/dist/xrpc.d.ts.map +1 -1
  205. package/dist/xrpc.js +255 -55
  206. package/package.json +21 -9
  207. package/public/admin-auth.js +271 -306
  208. package/public/admin.html +177 -61
  209. package/dist/db.d.ts.map +0 -1
  210. package/dist/db.js +0 -1361
  211. package/dist/fts.d.ts.map +0 -1
  212. package/dist/oauth/hooks.d.ts +0 -10
  213. package/dist/oauth/hooks.d.ts.map +0 -1
  214. package/dist/oauth/hooks.js +0 -40
  215. package/dist/schema.d.ts.map +0 -1
  216. package/dist/test-browser.d.ts +0 -14
  217. package/dist/test-browser.d.ts.map +0 -1
  218. package/dist/test-browser.js +0 -26
@@ -0,0 +1,559 @@
1
+ /**
2
+ * Keeping the index level with a permissioned space.
3
+ *
4
+ * A space never reaches a firehose. There is no stream to tail and no relay
5
+ * that has seen it, so the only way to know a space has moved is to ask the
6
+ * authority who has written into it and then ask each of those writers' hosts
7
+ * what changed. Two loops do that:
8
+ *
9
+ * reconcile the authority's `listRepos` gives the writer set with each
10
+ * writer's current rev. Anything whose rev has moved is synced;
11
+ * anything the authority no longer names is dropped.
12
+ * sync one writer's repo, forward from the rev we last read, through
13
+ * `listRepoOps` — or wholesale through `listRecords` when there
14
+ * is no rev to start from, or when the op log has been compacted
15
+ * past it.
16
+ *
17
+ * Write notices exist too and are far cheaper, but they are best-effort by
18
+ * design: the reference host forwards them once from an in-memory queue and
19
+ * logs the failure. So reconcile is the thing that makes sync correct, and a
20
+ * notice only makes it prompt.
21
+ *
22
+ * What is deliberately not done here is verification. The alpha signs a commit
23
+ * over the repo's LtHash state, and a syncer can check it. hatk does not check
24
+ * the equivalent on the firehose path either — it reads records out of relayed
25
+ * CAR blocks and trusts them — so checking here would claim a guarantee the
26
+ * rest of the index does not make. Adding it means depending on `@atproto/space`
27
+ * for LtHash and the commit verifier, and is worth doing to both paths at once
28
+ * rather than to this one alone.
29
+ */
30
+ import { getLexicon, getLexiconArray } from "../database/schema.js";
31
+ import { bulkInsertRecords, deleteRecord, insertRecord, purgeSpaceRecords } from "../database/db.js";
32
+ import { validateRecord } from '@bigmoves/lexicon';
33
+ import { noteReferencedRepos, trackRepo } from "../indexer.js";
34
+ import { emit, timer } from "../logger.js";
35
+ import { isPrivateCollection } from "../private-collections.js";
36
+ import { listSessionDids } from "../oauth/db.js";
37
+ import { forgetSpaceCredential, getSpaceCredential, isSpaceGone } from "./credential.js";
38
+ import { repoEndpoint, spaceHostEndpoint } from "./identity.js";
39
+ import { deleteSpaceRepo, deleteSpaceWatch, getSpaceRepo, getSpaceWatch, listSpaceRepos, listSpaceWatches, putSpaceRepo, putSpaceWatch, updateSpaceWatch, } from "./store.js";
40
+ import { parseSpaceRef, spaceRecordUri } from "./uri.js";
41
+ const OPS_PAGE = 1000;
42
+ const RECORDS_PAGE = 100;
43
+ /**
44
+ * Re-register this long before a registration lapses.
45
+ *
46
+ * The reference host holds one for a day, and a lapsed registration stops
47
+ * notices silently — the sweep would carry on working and nobody would notice
48
+ * the latency had gone back up.
49
+ */
50
+ const REGISTRATION_RENEW_LEAD_MS = 60 * 60 * 1000;
51
+ let options = null;
52
+ export function configureSpaceEngine(opts) {
53
+ options = opts;
54
+ }
55
+ function requireOptions() {
56
+ if (!options)
57
+ throw new Error('Space engine is not configured');
58
+ return options;
59
+ }
60
+ /**
61
+ * The collections a space of this type holds, narrowed to the ones hatk has a
62
+ * table for.
63
+ *
64
+ * A space type's lexicon declares its collections, which is what makes a space
65
+ * readable without the app being told what is in it — but a space may name
66
+ * collections this instance does not index, and those are simply not its
67
+ * business.
68
+ */
69
+ export function collectionsForSpaceType(spaceType) {
70
+ const { collections } = requireOptions();
71
+ const lexicon = getLexicon(spaceType);
72
+ const declared = lexicon?.defs?.main?.collections;
73
+ if (!Array.isArray(declared))
74
+ return [];
75
+ return declared
76
+ .filter((c) => typeof c === 'string')
77
+ .filter((c) => collections.has(c) && !isPrivateCollection(c));
78
+ }
79
+ // --- Credentials ---
80
+ /**
81
+ * Sessions worth trying for this space, best first.
82
+ *
83
+ * The reader that worked last time leads: a member who could read the space an
84
+ * hour ago is overwhelmingly likely to still be one, and putting them first
85
+ * means the common case costs one delegation rather than a walk through
86
+ * everybody signed in.
87
+ */
88
+ async function readerCandidates(watch) {
89
+ const dids = await listSessionDids();
90
+ if (!watch.readerDid)
91
+ return dids;
92
+ return [watch.readerDid, ...dids.filter((did) => did !== watch.readerDid)];
93
+ }
94
+ async function credentialFor(watch, refresh = false) {
95
+ const { oauth } = requireOptions();
96
+ const candidates = await readerCandidates(watch);
97
+ const credential = await getSpaceCredential(oauth, watch.space, candidates, { refresh });
98
+ if (credential.readerDid !== watch.readerDid) {
99
+ await updateSpaceWatch(watch.space, { readerDid: credential.readerDid });
100
+ watch.readerDid = credential.readerDid;
101
+ }
102
+ return credential;
103
+ }
104
+ async function credentialedGet(credential, endpoint, nsid, params) {
105
+ const url = new URL(`${endpoint}/xrpc/${nsid}`);
106
+ for (const [key, value] of Object.entries(params)) {
107
+ if (value !== undefined)
108
+ url.searchParams.set(key, String(value));
109
+ }
110
+ const res = await credential.fetch(url);
111
+ if (!res.ok) {
112
+ const body = (await res.json().catch(() => ({})));
113
+ const err = new Error(body.message ?? `${nsid} failed (${res.status})`);
114
+ err.status = res.status;
115
+ err.code = body.error;
116
+ throw err;
117
+ }
118
+ return (await res.json());
119
+ }
120
+ function statusOf(err) {
121
+ const status = err?.status;
122
+ return typeof status === 'number' ? status : undefined;
123
+ }
124
+ function codeOf(err) {
125
+ const code = err?.code;
126
+ return typeof code === 'string' ? code : undefined;
127
+ }
128
+ /**
129
+ * Run a credentialed read, re-minting once on a 401.
130
+ *
131
+ * A credential lives two hours and a sweep can outlive one. Re-minting costs a
132
+ * round trip; treating the expiry as a failure costs the whole sync and leaves
133
+ * the space looking unreadable when nothing about the membership changed.
134
+ */
135
+ async function withCredential(watch, operation) {
136
+ let credential = await credentialFor(watch);
137
+ try {
138
+ return await operation(credential);
139
+ }
140
+ catch (err) {
141
+ if (statusOf(err) !== 401)
142
+ throw err;
143
+ forgetSpaceCredential(watch.space);
144
+ credential = await credentialFor(watch, true);
145
+ return operation(credential);
146
+ }
147
+ }
148
+ // --- Record application ---
149
+ /**
150
+ * Skip a record the lexicon refuses, the way the firehose path does — and say
151
+ * why, the way the firehose path does. A record silently skipped is a record
152
+ * that never appears with no trace of the reason, which is the worst kind of
153
+ * missing.
154
+ */
155
+ function indexable(uri, collection, record) {
156
+ if (!record || typeof record !== 'object') {
157
+ emit('spaces', 'validation_skip', { uri, collection, error: 'record is not an object' });
158
+ return false;
159
+ }
160
+ const typed = record;
161
+ // A record whose $type disagrees with the collection it was filed under is
162
+ // not the record the collection's table is shaped for.
163
+ if (typeof typed.$type === 'string' && typed.$type !== collection) {
164
+ emit('spaces', 'validation_skip', { uri, collection, error: `$type is ${typed.$type}` });
165
+ return false;
166
+ }
167
+ const problem = validateRecord(getLexiconArray(), collection, { $type: collection, ...typed });
168
+ if (problem) {
169
+ emit('spaces', 'validation_skip', { uri, collection, path: problem.path, error: problem.message });
170
+ return false;
171
+ }
172
+ return true;
173
+ }
174
+ /**
175
+ * Apply one page of ops to the index.
176
+ *
177
+ * Ops carry no action: a create has no `prev`, a delete no `cid`, an update
178
+ * both. And `value` is inlined only when it is still the record's current
179
+ * value, so an op superseded later in the same page arrives without one — which
180
+ * is why the final state per rkey is resolved here before anything is written,
181
+ * and why a final op still missing its value has to be fetched.
182
+ */
183
+ async function applyOps(watch, writer, ops, credential, writerHost) {
184
+ const indexed = new Set(collectionsForSpaceType(watch.spaceType));
185
+ // Last op wins per record: a page can create and then delete the same rkey,
186
+ // and only the end state belongs in the index.
187
+ const final = new Map();
188
+ for (const op of ops) {
189
+ if (!indexed.has(op.collection))
190
+ continue;
191
+ final.set(`${op.collection}/${op.rkey}`, op);
192
+ }
193
+ let inserted = 0;
194
+ let deleted = 0;
195
+ let skipped = 0;
196
+ for (const op of final.values()) {
197
+ const uri = spaceRecordUri(watch.space, writer, op.collection, op.rkey);
198
+ if (!op.cid) {
199
+ await deleteRecord(op.collection, uri);
200
+ deleted++;
201
+ continue;
202
+ }
203
+ let value = op.value;
204
+ if (!value) {
205
+ // Superseded within the page, or served metadata-only. Either way the
206
+ // current value is one round trip away and the alternative is an index
207
+ // that silently disagrees with the repo.
208
+ try {
209
+ const out = await credentialedGet(credential, writerHost, 'com.atproto.space.getRecord', {
210
+ space: watch.space,
211
+ repo: writer,
212
+ collection: op.collection,
213
+ rkey: op.rkey,
214
+ });
215
+ value = out.value;
216
+ }
217
+ catch (err) {
218
+ if (statusOf(err) === 404 || codeOf(err) === 'RecordNotFound') {
219
+ await deleteRecord(op.collection, uri);
220
+ deleted++;
221
+ continue;
222
+ }
223
+ throw err;
224
+ }
225
+ }
226
+ if (!indexable(uri, op.collection, value)) {
227
+ skipped++;
228
+ continue;
229
+ }
230
+ noteReferencedRepos(op.collection, op.rkey, value);
231
+ await insertRecord(op.collection, uri, op.cid, writer, { $type: op.collection, ...value });
232
+ inserted++;
233
+ }
234
+ return { inserted, deleted, skipped };
235
+ }
236
+ // --- Sync ---
237
+ /**
238
+ * Read one writer's repo forward from where we left it.
239
+ *
240
+ * Returns the rev the repo was read to, or null if the op log could not carry
241
+ * us there — the log is a transport optimization with no history guarantee, so
242
+ * a host may have compacted past our position, and the answer then is a full
243
+ * read rather than a gap.
244
+ */
245
+ async function syncForward(watch, writer, since, credential, writerHost) {
246
+ let cursor;
247
+ let head = null;
248
+ let totals = { inserted: 0, deleted: 0, skipped: 0 };
249
+ for (let page = 0; page < 100; page++) {
250
+ const out = await credentialedGet(credential, writerHost, 'com.atproto.space.listRepoOps', {
251
+ space: watch.space,
252
+ repo: writer,
253
+ since,
254
+ cursor,
255
+ limit: OPS_PAGE,
256
+ });
257
+ const ops = (out.ops ?? []);
258
+ const applied = await applyOps(watch, writer, ops, credential, writerHost);
259
+ totals = {
260
+ inserted: totals.inserted + applied.inserted,
261
+ deleted: totals.deleted + applied.deleted,
262
+ skipped: totals.skipped + applied.skipped,
263
+ };
264
+ // The commit describes the head, so it only appears once the response has
265
+ // reached it; a full page has more behind it and carries a cursor instead.
266
+ const commitRev = out.commit?.rev;
267
+ if (typeof commitRev === 'string') {
268
+ head = commitRev;
269
+ break;
270
+ }
271
+ cursor = typeof out.cursor === 'string' ? out.cursor : undefined;
272
+ if (!cursor)
273
+ break;
274
+ }
275
+ if (totals.inserted || totals.deleted || totals.skipped) {
276
+ emit('spaces', 'sync_ops', { space: watch.space, writer, ...totals, head });
277
+ }
278
+ return head;
279
+ }
280
+ /**
281
+ * Read a writer's whole repo in a space and replace what the index holds for it.
282
+ *
283
+ * The rev is taken before the read, not after: a write landing mid-read is then
284
+ * re-read on the next sweep rather than skipped, which is the direction to be
285
+ * wrong in.
286
+ */
287
+ async function syncFull(watch, writer, credential, writerHost) {
288
+ const collections = collectionsForSpaceType(watch.spaceType);
289
+ if (collections.length === 0)
290
+ return null;
291
+ let rev = null;
292
+ try {
293
+ const out = await credentialedGet(credential, writerHost, 'com.atproto.space.getLatestCommit', {
294
+ space: watch.space,
295
+ repo: writer,
296
+ });
297
+ rev = typeof out.commit?.rev === 'string' ? out.commit.rev : null;
298
+ }
299
+ catch (err) {
300
+ // A writer the authority named but whose host holds no repo for them yet:
301
+ // nothing to read, and nothing wrong.
302
+ if (statusOf(err) === 404 || codeOf(err) === 'RepoNotFound')
303
+ return null;
304
+ throw err;
305
+ }
306
+ const records = [];
307
+ let skipped = 0;
308
+ for (const collection of collections) {
309
+ let cursor;
310
+ for (let page = 0; page < 200; page++) {
311
+ let out;
312
+ try {
313
+ out = await credentialedGet(credential, writerHost, 'com.atproto.space.listRecords', {
314
+ space: watch.space,
315
+ repo: writer,
316
+ collection,
317
+ cursor,
318
+ limit: RECORDS_PAGE,
319
+ });
320
+ }
321
+ catch (err) {
322
+ // `RepoNotFound` here means this writer holds nothing in the space,
323
+ // which the lexicon notes does not distinguish a member who has never
324
+ // written from someone who is not one. Either way: no records.
325
+ if (statusOf(err) === 404 || codeOf(err) === 'RepoNotFound')
326
+ break;
327
+ throw err;
328
+ }
329
+ for (const rec of (out.records ?? [])) {
330
+ if (!indexable(spaceRecordUri(watch.space, writer, collection, rec.rkey), collection, rec.value)) {
331
+ skipped++;
332
+ continue;
333
+ }
334
+ noteReferencedRepos(collection, rec.rkey, rec.value);
335
+ records.push({
336
+ collection,
337
+ uri: spaceRecordUri(watch.space, writer, collection, rec.rkey),
338
+ cid: rec.cid,
339
+ did: writer,
340
+ record: { $type: collection, ...rec.value },
341
+ });
342
+ }
343
+ cursor = typeof out.cursor === 'string' ? out.cursor : undefined;
344
+ if (!cursor)
345
+ break;
346
+ }
347
+ }
348
+ await purgeSpaceRecords(watch.space, writer, collections);
349
+ const inserted = records.length > 0 ? await bulkInsertRecords(records) : 0;
350
+ emit('spaces', 'sync_full', { space: watch.space, writer, inserted, skipped, rev });
351
+ return rev;
352
+ }
353
+ /** Bring one writer's repo in a space up to date, however that has to happen. */
354
+ export async function syncSpaceRepo(watch, writer, credential) {
355
+ const writerHost = await repoEndpoint(writer);
356
+ const local = await getSpaceRepo(watch.space, writer);
357
+ let rev = null;
358
+ if (local?.rev) {
359
+ try {
360
+ rev = await syncForward(watch, writer, local.rev, credential, writerHost);
361
+ }
362
+ catch (err) {
363
+ if (statusOf(err) === 401)
364
+ throw err;
365
+ emit('spaces', 'incremental_fell_back', {
366
+ space: watch.space,
367
+ writer,
368
+ since: local.rev,
369
+ error: err instanceof Error ? err.message : String(err),
370
+ });
371
+ rev = null;
372
+ }
373
+ }
374
+ if (!rev)
375
+ rev = await syncFull(watch, writer, credential, writerHost);
376
+ if (!rev)
377
+ return;
378
+ await putSpaceRepo({ space: watch.space, did: writer, pds: writerHost, rev });
379
+ }
380
+ // --- Reconcile ---
381
+ /**
382
+ * Subscribe to this space's write notices, if this instance can receive them.
383
+ *
384
+ * Authenticated with the space credential, so only somebody the authority
385
+ * already admits can subscribe — which is why this happens inside reconcile,
386
+ * where a credential is already in hand. The delivery endpoint is not sent:
387
+ * the authority resolves it from our own DID document, so a registration can
388
+ * only ever point at an endpoint we published for ourselves.
389
+ *
390
+ * Best-effort. Failing to register costs latency, not correctness, and must
391
+ * not fail the sweep that was about to read the space anyway.
392
+ */
393
+ async function ensureRegistered(watch, credential) {
394
+ const { serviceId } = requireOptions();
395
+ if (!serviceId)
396
+ return;
397
+ const until = watch.registeredUntil ? Date.parse(watch.registeredUntil) : 0;
398
+ if (Number.isFinite(until) && until - REGISTRATION_RENEW_LEAD_MS > Date.now())
399
+ return;
400
+ try {
401
+ const authorityHost = await spaceHostEndpoint(watch.authority);
402
+ const res = await credential.fetch(`${authorityHost}/xrpc/com.atproto.space.registerNotify`, {
403
+ method: 'POST',
404
+ headers: { 'Content-Type': 'application/json' },
405
+ body: JSON.stringify({ space: watch.space, service: serviceId }),
406
+ });
407
+ if (!res.ok)
408
+ throw new Error(`registerNotify refused (${res.status})`);
409
+ const body = (await res.json());
410
+ if (body.expiresAt) {
411
+ await updateSpaceWatch(watch.space, { registeredUntil: body.expiresAt });
412
+ watch.registeredUntil = body.expiresAt;
413
+ }
414
+ emit('spaces', 'registered', { space: watch.space, expires_at: body.expiresAt });
415
+ }
416
+ catch (err) {
417
+ emit('spaces', 'register_failed', {
418
+ space: watch.space,
419
+ error: err instanceof Error ? err.message : String(err),
420
+ });
421
+ }
422
+ }
423
+ async function listWriters(watch, credential) {
424
+ const authorityHost = await spaceHostEndpoint(watch.authority);
425
+ const writers = [];
426
+ let cursor;
427
+ for (let page = 0; page < 100; page++) {
428
+ const out = await credentialedGet(credential, authorityHost, 'com.atproto.space.listRepos', {
429
+ space: watch.space,
430
+ cursor,
431
+ limit: 1000,
432
+ });
433
+ for (const repo of (out.repos ?? [])) {
434
+ writers.push({ did: repo.did, rev: typeof repo.rev === 'string' ? repo.rev : null });
435
+ }
436
+ cursor = typeof out.cursor === 'string' ? out.cursor : undefined;
437
+ if (!cursor)
438
+ break;
439
+ }
440
+ return writers;
441
+ }
442
+ /**
443
+ * Compare the authority's writer set against what the index holds, and close
444
+ * the difference.
445
+ *
446
+ * The writer set is what the authority has been told, not what is true: a
447
+ * repo's own host is the source of truth, and after a community migrates hosts
448
+ * the set is empty until each writer writes again. So a writer the authority
449
+ * stops naming has their rows dropped, but the rev comparison is only ever a
450
+ * reason to read — never a reason to believe a repo has not moved.
451
+ */
452
+ export async function reconcileSpace(watch) {
453
+ const elapsed = timer();
454
+ await withCredential(watch, async (credential) => {
455
+ await ensureRegistered(watch, credential);
456
+ const writers = await listWriters(watch, credential);
457
+ const remote = new Set(writers.map((w) => w.did));
458
+ // Whoever writes into the space is somebody the app will name, so their
459
+ // public repo — profile first of all — is indexed too.
460
+ for (const writer of writers)
461
+ trackRepo(writer.did);
462
+ let synced = 0;
463
+ for (const writer of writers) {
464
+ const local = await getSpaceRepo(watch.space, writer.did);
465
+ if (local?.rev && writer.rev && local.rev === writer.rev)
466
+ continue;
467
+ await syncSpaceRepo(watch, writer.did, credential);
468
+ synced++;
469
+ }
470
+ // A writer the space no longer names is not readable through it any more,
471
+ // so neither are their rows.
472
+ let dropped = 0;
473
+ const collections = collectionsForSpaceType(watch.spaceType);
474
+ for (const local of await listSpaceRepos(watch.space)) {
475
+ if (remote.has(local.did))
476
+ continue;
477
+ await purgeSpaceRecords(watch.space, local.did, collections);
478
+ await deleteSpaceRepo(watch.space, local.did);
479
+ dropped++;
480
+ }
481
+ await updateSpaceWatch(watch.space, { lastError: null });
482
+ emit('spaces', 'reconcile', {
483
+ space: watch.space,
484
+ writers: writers.length,
485
+ synced,
486
+ dropped,
487
+ reader_did: watch.readerDid,
488
+ duration_ms: elapsed(),
489
+ });
490
+ });
491
+ }
492
+ // --- Public surface ---
493
+ /** Follow a space from now on, and read it once. */
494
+ export async function watchSpace(space) {
495
+ const { types } = requireOptions();
496
+ const ref = parseSpaceRef(space);
497
+ if (!ref)
498
+ throw new Error(`Not a space ref: ${space}`);
499
+ if (!types.has(ref.type))
500
+ throw new Error(`Not an indexed space type: ${ref.type}`);
501
+ await putSpaceWatch({ space, authority: ref.authority, spaceType: ref.type });
502
+ const watch = (await getSpaceWatch(space));
503
+ await reconcileSpace(watch);
504
+ return watch;
505
+ }
506
+ /** Stop following a space and drop everything indexed from it. */
507
+ export async function unwatchSpace(space) {
508
+ const watch = await getSpaceWatch(space);
509
+ if (watch) {
510
+ const collections = collectionsForSpaceType(watch.spaceType);
511
+ for (const repo of await listSpaceRepos(space)) {
512
+ await purgeSpaceRecords(space, repo.did, collections);
513
+ }
514
+ }
515
+ await deleteSpaceWatch(space);
516
+ forgetSpaceCredential(space);
517
+ emit('spaces', 'unwatched', { space });
518
+ }
519
+ /**
520
+ * Act on a write notice: read the one repo it names.
521
+ *
522
+ * Only for a space already followed. A notice about anything else is not an
523
+ * instruction to start following it — that decision belongs to config or to the
524
+ * app, never to whoever can reach this endpoint.
525
+ *
526
+ * The notice's `rev` is deliberately not trusted as the new position: it is
527
+ * read from the repo's own host, which is the source of truth, by the ordinary
528
+ * sync path. The notice only says "look again".
529
+ */
530
+ export async function handleWriteNotice(notice) {
531
+ const watch = await getSpaceWatch(notice.space);
532
+ if (!watch)
533
+ return false;
534
+ await withCredential(watch, (credential) => syncSpaceRepo(watch, notice.repo, credential));
535
+ return true;
536
+ }
537
+ /**
538
+ * Reconcile every followed space.
539
+ *
540
+ * Failures are per space and recorded rather than thrown: one community's host
541
+ * being unreachable is not a reason to stop reading the others, and a space
542
+ * whose last reader logged out should go quiet rather than take the sweep down.
543
+ */
544
+ export async function reconcileAll() {
545
+ for (const watch of await listSpaceWatches()) {
546
+ try {
547
+ await reconcileSpace(watch);
548
+ }
549
+ catch (err) {
550
+ if (isSpaceGone(err)) {
551
+ await unwatchSpace(watch.space);
552
+ continue;
553
+ }
554
+ const message = err instanceof Error ? err.message : String(err);
555
+ await updateSpaceWatch(watch.space, { lastError: message });
556
+ emit('spaces', 'reconcile_failed', { space: watch.space, error: message });
557
+ }
558
+ }
559
+ }
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Refusing hand-written SQL that would serve a private space to everyone.
3
+ *
4
+ * Every read hatk owns applies the space gate. The one it does not own is raw
5
+ * SQL from an app's own handlers — a feed, a query, a hydration step — and
6
+ * nothing can inject a predicate into a string somebody else wrote. So the
7
+ * open question was who remembers to add it. The honest answer is nobody: a
8
+ * feed written today over public data becomes a leak the day somebody adds a
9
+ * space type that writes into the same collection, and the feed never changed.
10
+ *
11
+ * This turns that silent leak into a loud, deterministic error. hatk knows at
12
+ * boot exactly which tables can hold a space row — the collections the
13
+ * configured space types declare — so a raw query naming one of those tables
14
+ * without the gate is refused, by name, with the helper that fixes it.
15
+ *
16
+ * A guard rail rather than the boundary. Detection is a token match on the
17
+ * quoted table name and on the gate's own SQL; it is meant to catch the
18
+ * mistake in development on the first run, not to survive an adversary. The
19
+ * gate applied by the helpers is the boundary. On an instance that indexes no
20
+ * space the set is empty and this is a Set lookup that never matches.
21
+ */
22
+ export declare function setSpaceBackedCollections(collections: Iterable<string>): void;
23
+ export declare function spaceBackedCollections(): ReadonlySet<string>;
24
+ export declare class UngatedSpaceQueryError extends Error {
25
+ readonly table: string;
26
+ constructor(table: string);
27
+ }
28
+ /** Throw if `sql` names a space-backed table and carries no gate. */
29
+ export declare function assertGatedSql(sql: string): void;
30
+ /** `querySQL` for app-facing contexts: the same query, refused when it would leak. */
31
+ export declare function guardedQuerySQL(sql: string, params?: unknown[]): Promise<unknown[]>;
32
+ /**
33
+ * `querySQL` with the guard deliberately off.
34
+ *
35
+ * Its own name rather than an option, so that every place an app reads a
36
+ * space-backed table without the gate says so where it does it and can be
37
+ * found with grep. For results that are never served to a viewer: an admin
38
+ * rollup, a count, a maintenance pass.
39
+ */
40
+ export declare function unfilteredQuerySQL(sql: string, params?: unknown[]): Promise<unknown[]>;
41
+ //# sourceMappingURL=guard.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"guard.d.ts","sourceRoot":"","sources":["../../src/spaces/guard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAOH,wBAAgB,yBAAyB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,CAE7E;AAED,wBAAgB,sBAAsB,IAAI,WAAW,CAAC,MAAM,CAAC,CAE5D;AAYD,qBAAa,sBAAuB,SAAQ,KAAK;IACnC,QAAQ,CAAC,KAAK,EAAE,MAAM;gBAAb,KAAK,EAAE,MAAM;CASnC;AAED,qEAAqE;AACrE,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAMhD;AAED,sFAAsF;AACtF,wBAAsB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAK7F;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,OAAO,EAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAE1F"}
@@ -0,0 +1,79 @@
1
+ /**
2
+ * Refusing hand-written SQL that would serve a private space to everyone.
3
+ *
4
+ * Every read hatk owns applies the space gate. The one it does not own is raw
5
+ * SQL from an app's own handlers — a feed, a query, a hydration step — and
6
+ * nothing can inject a predicate into a string somebody else wrote. So the
7
+ * open question was who remembers to add it. The honest answer is nobody: a
8
+ * feed written today over public data becomes a leak the day somebody adds a
9
+ * space type that writes into the same collection, and the feed never changed.
10
+ *
11
+ * This turns that silent leak into a loud, deterministic error. hatk knows at
12
+ * boot exactly which tables can hold a space row — the collections the
13
+ * configured space types declare — so a raw query naming one of those tables
14
+ * without the gate is refused, by name, with the helper that fixes it.
15
+ *
16
+ * A guard rail rather than the boundary. Detection is a token match on the
17
+ * quoted table name and on the gate's own SQL; it is meant to catch the
18
+ * mistake in development on the first run, not to survive an adversary. The
19
+ * gate applied by the helpers is the boundary. On an instance that indexes no
20
+ * space the set is empty and this is a Set lookup that never matches.
21
+ */
22
+ import { querySQL } from "../database/db.js";
23
+ /** Tables that can hold a row from a permissioned space. Empty unless spaces are configured. */
24
+ let spaceBacked = new Set();
25
+ export function setSpaceBackedCollections(collections) {
26
+ spaceBacked = new Set(collections);
27
+ }
28
+ export function spaceBackedCollections() {
29
+ return spaceBacked;
30
+ }
31
+ /**
32
+ * The gate's own predicate, in either of the shapes `spaceFilterSql` emits:
33
+ * `t.space IS NULL` when nothing is readable, and
34
+ * `(t.space IS NULL OR t.space IN (...))` when something is. A query that
35
+ * selects one space with `t.space = $1` does not contain this and is refused —
36
+ * correctly, because naming a space is not the same as being allowed to read
37
+ * it.
38
+ */
39
+ const GATE_TOKEN = /\bspace\s+is\s+null\b/i;
40
+ export class UngatedSpaceQueryError extends Error {
41
+ table;
42
+ constructor(table) {
43
+ super(`Raw SQL reads "${table}", which permissioned spaces write into, without the space gate. ` +
44
+ `Records in it are private to the space's members. Apply ctx.spaceFilter(alias, nextParam) ` +
45
+ `to the query, read through ctx.paginate or the typed helpers, or use ctx.db.unfiltered ` +
46
+ `if this result is never served to a viewer.`);
47
+ this.table = table;
48
+ this.name = 'UngatedSpaceQueryError';
49
+ }
50
+ }
51
+ /** Throw if `sql` names a space-backed table and carries no gate. */
52
+ export function assertGatedSql(sql) {
53
+ if (spaceBacked.size === 0)
54
+ return;
55
+ if (GATE_TOKEN.test(sql))
56
+ return;
57
+ for (const collection of spaceBacked) {
58
+ if (sql.includes(`"${collection}"`))
59
+ throw new UngatedSpaceQueryError(collection);
60
+ }
61
+ }
62
+ /** `querySQL` for app-facing contexts: the same query, refused when it would leak. */
63
+ export async function guardedQuerySQL(sql, params = []) {
64
+ // Async so a refusal is a rejected promise like any other query failure,
65
+ // rather than a synchronous throw a caller awaiting the result never sees.
66
+ assertGatedSql(sql);
67
+ return querySQL(sql, params);
68
+ }
69
+ /**
70
+ * `querySQL` with the guard deliberately off.
71
+ *
72
+ * Its own name rather than an option, so that every place an app reads a
73
+ * space-backed table without the gate says so where it does it and can be
74
+ * found with grep. For results that are never served to a viewer: an admin
75
+ * rollup, a count, a maintenance pass.
76
+ */
77
+ export function unfilteredQuerySQL(sql, params = []) {
78
+ return querySQL(sql, params);
79
+ }