@estiva-app/interop 0.92.0 → 0.94.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,396 @@
1
+ /**
2
+ * A topic's files in the order and look someone gave them (SPEC §6.14).
3
+ *
4
+ * The topic keeps, per listed file, two fields beside the entry that lists it:
5
+ * `order:<d>`, a sort key that places the file between its neighbours, and
6
+ * `look:<d>`, `row` or `card`. Each is a `kind:1851` on the topic with §6.14's
7
+ * tags and skip rules, `["o", <d>]` included, so two people arranging at once
8
+ * both keep their change.
9
+ *
10
+ * - Files with a place come first, by key compared byte-wise, equal keys by
11
+ * the lower id; files with none follow, oldest listed first (§6.3).
12
+ * - A place older than the file's newest `lists:<d>` = `true` is ignored: a
13
+ * file taken off and put back comes back unplaced, unless the put-back
14
+ * places it again ({@link planPutBackInTopic}).
15
+ * - A missing or unknown look draws the manifest's widget: the look is a hint.
16
+ * - Every `order:` change names the change it follows (`prev`), so a tab that
17
+ * read the topic before someone else moved the file is refused, and plans
18
+ * the move again on what is there now ({@link writeArrangement}).
19
+ *
20
+ * why: https://github.com/estiva-app/estiva-docs/blob/main/decisions/each-file-on-a-topic-keeps-its-own-place-and-look.md
21
+ */
22
+ import type { SignedEvent, UnsignedEvent } from '@estiva-app/protocol'
23
+ import { byOrder } from '@estiva-app/conversation'
24
+ import { changedAt, combineChanges, conflictOf, fieldsSetBy } from './fields.js'
25
+
26
+ const KIND_CHANGE = 1851
27
+ const ADDRESS = /^(\d+):([0-9a-f]{64}):(.+)$/
28
+
29
+ export type FileLook = 'row' | 'card'
30
+
31
+ const ORDER = 'order:'
32
+ const LOOK = 'look:'
33
+ /** The field a file's place on a topic is kept in: `order:<d>`. */
34
+ export const orderField = (file: string): string => `${ORDER}${file}`
35
+ /** The field a file's look on a topic is kept in: `look:<d>`. */
36
+ export const lookField = (file: string): string => `${LOOK}${file}`
37
+
38
+ /** Digits in byte order: `0`–`9`, `A`–`Z`, `a`–`z`. */
39
+ const DIGITS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
40
+ const KEY = /^[0-9A-Za-z]*[1-9A-Za-z]$/
41
+ /** The longest key a reader takes (§6.14); a planner whose key would pass it places every file afresh. */
42
+ export const MAX_PLACE_KEY = 64
43
+
44
+ /** A place §6.14 reads: 1 to 64 of `0-9A-Za-z`, not ending in `0`. Anything else is no place. */
45
+ export const isPlaceKey = (key: string | undefined): key is string =>
46
+ typeof key === 'string' && key.length <= MAX_PLACE_KEY && KEY.test(key)
47
+
48
+ /** `n` keys, evenly spread and as short as `n` allows, in order: every file placed afresh. */
49
+ function evenKeys(n: number): string[] {
50
+ let width = 1
51
+ while (DIGITS.length ** width < (n + 1) * 2) width++
52
+ const span = DIGITS.length ** width
53
+ return Array.from({ length: n }, (_, i) => {
54
+ let value = Math.floor(((i + 1) * span) / (n + 1))
55
+ let key = ''
56
+ for (let w = 0; w < width; w++) {
57
+ key = DIGITS[value % DIGITS.length] + key
58
+ value = Math.floor(value / DIGITS.length)
59
+ }
60
+ // Never ending in `0`: a trailing `1` still sorts after this key and before the next.
61
+ return key.endsWith('0') ? `${key}1` : key
62
+ })
63
+ }
64
+
65
+ /** Byte-wise, as §6.14 compares keys (never `localeCompare`). */
66
+ const compareKeys = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0)
67
+
68
+ /**
69
+ * A key that sorts strictly between `before` and `after`, either left out for
70
+ * an open end. Each must be a valid key and `before` sort before `after`.
71
+ * Pure, so two people dropping files between the same neighbours get the same
72
+ * key; the file's id breaks that tie.
73
+ */
74
+ export function keyBetween(before?: string, after?: string): string {
75
+ if (before !== undefined && !isPlaceKey(before)) throw new Error(`not a place key: ${JSON.stringify(before)}`)
76
+ if (after !== undefined && !isPlaceKey(after)) throw new Error(`not a place key: ${JSON.stringify(after)}`)
77
+ if (before !== undefined && after !== undefined && compareKeys(before, after) >= 0) {
78
+ throw new Error(`${JSON.stringify(before)} does not sort before ${JSON.stringify(after)}`)
79
+ }
80
+ return midpoint(before ?? '', after)
81
+ }
82
+
83
+ /** Between `a` (possibly empty) and `b` (absent: no upper end), neither ending in `0`. */
84
+ function midpoint(a: string, b: string | undefined): string {
85
+ if (b !== undefined) {
86
+ // A shared start stays; the rest is placed between what follows it.
87
+ let n = 0
88
+ while ((a[n] ?? '0') === b[n]) n++
89
+ if (n > 0) return b.slice(0, n) + midpoint(a.slice(n), b.slice(n))
90
+ }
91
+ const low = a ? DIGITS.indexOf(a[0]) : 0
92
+ const high = b !== undefined ? DIGITS.indexOf(b[0]) : DIGITS.length
93
+ if (high - low > 1) return DIGITS[Math.round((low + high) / 2)]
94
+ // Adjacent digits: a longer `b` has room under its first digit; otherwise go one digit deeper after `a`'s.
95
+ if (b !== undefined && b.length > 1) return b.slice(0, 1)
96
+ return DIGITS[low] + midpoint(a.slice(1), undefined)
97
+ }
98
+
99
+ /** What a reader knows of one listed file's place and look, beside its list entry. */
100
+ export interface FileArrangement {
101
+ /** Its place, when it has one that counts: a valid key no older than its newest listing. */
102
+ order?: string
103
+ /** Its look, when set to one a reader knows; otherwise draw the manifest's widget. */
104
+ look?: FileLook
105
+ /**
106
+ * What the next change of each field follows: the field's newest change
107
+ * on the relay, one a reader skips included (the relay chains them all),
108
+ * or the topic's first record when the field never changed.
109
+ */
110
+ prev: { order: string; look: string }
111
+ /** The latest time (ms) among those, so a change is stamped after it. */
112
+ prevAt: number
113
+ }
114
+
115
+ /** A file in the order it is drawn, as the planners are handed it. */
116
+ export interface ArrangedFile extends FileArrangement {
117
+ /** The listed file's id. */
118
+ file: string
119
+ }
120
+
121
+ interface Heads {
122
+ /** Newest good change per field, by field name. */
123
+ good: Map<string, SignedEvent>
124
+ /** Newest change per field, any, by field name: the next `prev`. */
125
+ any: Map<string, SignedEvent>
126
+ }
127
+
128
+ const tagOf = (event: Pick<SignedEvent, 'tags'>, name: string): string | undefined =>
129
+ event.tags.find((t) => t[0] === name && t[1])?.[1]
130
+
131
+ const isArrangeField = (field: string) => field.startsWith(ORDER) || field.startsWith(LOOK)
132
+ /** The listed file a per-file field of a list names — `lists:`, `order:` or `look:` — or `null` for any other field. */
133
+ export const fileOfListField = (field: string): string | null =>
134
+ /^(lists|order|look):./.test(field) ? field.slice(field.indexOf(':') + 1) : null
135
+ const fileOfField = (field: string) => fileOfListField(field) ?? ''
136
+
137
+ /**
138
+ * Whether a change setting `field` on `list` is one a reader folds (§6.14's
139
+ * skip rules, for `lists:`, `order:` and `look:` alike): one of its `o` tags
140
+ * is the field's `<d>`, its `a` is the list, its `O` the list's id, the `<d>`
141
+ * is not the list itself, and its `h` is the list's team.
142
+ */
143
+ export function readsAs(event: SignedEvent, field: string, list: string, team: string | null | undefined): boolean {
144
+ const holder = ADDRESS.exec(list)
145
+ const file = fileOfField(field)
146
+ return (
147
+ !!holder &&
148
+ !!file &&
149
+ event.tags.some((t) => t[0] === 'o' && t[1] === file) &&
150
+ tagOf(event, 'a') === list &&
151
+ tagOf(event, 'O') === holder[3] &&
152
+ file !== holder[3] &&
153
+ !!team &&
154
+ tagOf(event, 'h') === team
155
+ )
156
+ }
157
+
158
+ /** The `order:` and `look:` changes per list, newest good and newest of any. */
159
+ export function arrangementHeads(
160
+ events: readonly SignedEvent[],
161
+ teamOfList: (list: string) => string | null | undefined,
162
+ ): Map<string, Heads> {
163
+ const out = new Map<string, Heads>()
164
+ for (const event of events) {
165
+ if (event.kind !== KIND_CHANGE) continue
166
+ const list = tagOf(event, 'a')
167
+ if (!list) continue
168
+ for (const { field } of fieldsSetBy(event)) {
169
+ if (!isArrangeField(field)) continue
170
+ const heads = out.get(list) ?? { good: new Map(), any: new Map() }
171
+ out.set(list, heads)
172
+ const held = heads.any.get(field)
173
+ if (!held || byOrder(held, event) < 0) heads.any.set(field, event)
174
+ if (!readsAs(event, field, list, teamOfList(list))) continue
175
+ const good = heads.good.get(field)
176
+ if (!good || byOrder(good, event) < 0) heads.good.set(field, event)
177
+ }
178
+ }
179
+ return out
180
+ }
181
+
182
+ const valueIn = (event: SignedEvent, field: string) => fieldsSetBy(event).find((f) => f.field === field)?.value
183
+
184
+ /**
185
+ * One file's arrangement on a list, from {@link arrangementHeads}: `listed`
186
+ * is the file's newest `lists:<d>` = `true` entry, `root` the list's first
187
+ * record (the first `prev`).
188
+ */
189
+ export function arrangementOf(heads: Heads | undefined, file: string, listed: SignedEvent, root: SignedEvent): FileArrangement {
190
+ const placed = heads?.good.get(orderField(file))
191
+ const key = placed && byOrder(placed, listed) >= 0 ? valueIn(placed, orderField(file)) : undefined
192
+ const looked = heads?.good.get(lookField(file))
193
+ const look = looked ? valueIn(looked, lookField(file)) : undefined
194
+ const orderHead = heads?.any.get(orderField(file))
195
+ const lookHead = heads?.any.get(lookField(file))
196
+ return {
197
+ ...(isPlaceKey(key) ? { order: key } : {}),
198
+ ...(look === 'row' || look === 'card' ? { look } : {}),
199
+ prev: { order: orderHead?.id ?? root.id, look: lookHead?.id ?? root.id },
200
+ prevAt: Math.max(changedAt(root), orderHead ? changedAt(orderHead) : 0, lookHead ? changedAt(lookHead) : 0),
201
+ }
202
+ }
203
+
204
+ /**
205
+ * §6.14's order, stable: placed files by key (equal keys by the lower id),
206
+ * then the rest in the order given, which is oldest listed first.
207
+ */
208
+ export function sortArranged<T extends { file: string; order?: string }>(files: readonly T[]): T[] {
209
+ const placed = files.filter((f) => f.order !== undefined)
210
+ const rest = files.filter((f) => f.order === undefined)
211
+ placed.sort((a, b) => compareKeys(a.order!, b.order!) || compareKeys(a.file, b.file))
212
+ return [...placed, ...rest]
213
+ }
214
+
215
+ /** The topic the change goes on: its address and its team, read from its root (§6.2). */
216
+ export interface ArrangeHolder {
217
+ address: string
218
+ folder: string
219
+ }
220
+
221
+ /** One field's change on the topic: §6.2's one-field tag order, then `o`. */
222
+ function fieldChange(pubkey: string, createdAtMs: number, topic: ArrangeHolder, field: string, value: string, prev: string): UnsignedEvent {
223
+ const holder = ADDRESS.exec(topic.address)
224
+ if (!holder) throw new Error(`not a file address: ${JSON.stringify(topic.address)}`)
225
+ if (!topic.folder) throw new Error("a change names the team of the file it is on")
226
+ const file = fileOfField(field)
227
+ if (file === holder[3]) throw new Error('a file is not on its own list')
228
+ return {
229
+ pubkey,
230
+ created_at: Math.floor(createdAtMs / 1000),
231
+ kind: KIND_CHANGE,
232
+ tags: [
233
+ ['a', topic.address],
234
+ ['O', holder[3]],
235
+ ['field', field],
236
+ ['value', value],
237
+ ['h', topic.folder],
238
+ ['ts', String(createdAtMs)],
239
+ ['prev', prev],
240
+ ['o', file],
241
+ ],
242
+ content: '',
243
+ }
244
+ }
245
+
246
+ /** Stamped after every change it follows, so a clock that is behind is not refused. */
247
+ const stamp = (createdAtMs: number, files: readonly FileArrangement[]) => Math.max(createdAtMs, ...files.map((f) => f.prevAt + 1))
248
+
249
+ /**
250
+ * Move `file` to just after `after` (`null`: first) in `files`, which is the
251
+ * topic's files in the order drawn. One change: the moved file's place, and
252
+ * where it lands among files with no place, the places of those above it too,
253
+ * since unplaced files follow placed ones; files tied on one key with the one
254
+ * it lands before are placed again after it. Several places go in one change,
255
+ * taken whole or refused whole. `null` when the file is already there.
256
+ */
257
+ export function planMoveInTopic(
258
+ pubkey: string,
259
+ createdAtMs: number,
260
+ args: { topic: ArrangeHolder; files: readonly ArrangedFile[]; file: string; after: string | null },
261
+ ): UnsignedEvent | null {
262
+ // In the topic's own order, whatever order the app draws them in (Ship groups by kind).
263
+ const files = sortArranged(args.files)
264
+ const moved = files.find((f) => f.file === args.file)
265
+ if (!moved) throw new Error(`${args.file} is not on the topic's list`)
266
+ const rest = files.filter((f) => f.file !== args.file)
267
+ const to = args.after === null ? 0 : rest.findIndex((f) => f.file === args.after) + 1
268
+ if (args.after !== null && to === 0) throw new Error(`${args.after} is not on the topic's list`)
269
+ // Already there: nothing to write.
270
+ if (files.indexOf(moved) === to) return null
271
+ const before = rest.slice(0, to)
272
+ const after = rest.slice(to)
273
+ const writes: { file: ArrangedFile; key: string }[] = []
274
+ const unplaced = before.findIndex((f) => f.order === undefined)
275
+ let low = (unplaced < 0 ? before.at(-1) : before[unplaced - 1])?.order
276
+ const place = (file: ArrangedFile, high?: string) => {
277
+ const key = keyBetween(low, high)
278
+ writes.push({ file, key })
279
+ low = key
280
+ }
281
+ if (unplaced >= 0) {
282
+ // Everything after an unplaced file is unplaced too: nothing bounds the keys above.
283
+ for (const f of before.slice(unplaced)) place(f)
284
+ place(moved)
285
+ } else if (after[0]?.order !== undefined && after[0].order === low) {
286
+ // It lands between two files sharing one key: place it, and the tied ones after it, below the next key.
287
+ let j = 0
288
+ while (after[j]?.order === low) j++
289
+ const high = after[j]?.order
290
+ place(moved, high)
291
+ for (const f of after.slice(0, j)) place(f, high)
292
+ } else {
293
+ place(moved, after[0]?.order)
294
+ }
295
+ if (writes.some((w) => w.key.length > MAX_PLACE_KEY)) {
296
+ // Keys grew too long (many moves to one spot): place every placed file, and the moved one, afresh.
297
+ const placed = [...before, moved, ...after].filter((f) => f === moved || f.order !== undefined || before.includes(f))
298
+ const keys = evenKeys(placed.length)
299
+ writes.splice(0, writes.length, ...placed.map((file, i) => ({ file, key: keys[i] })))
300
+ }
301
+ const at = stamp(createdAtMs, writes.map((w) => w.file))
302
+ const changes = writes.map((w) => fieldChange(pubkey, at, args.topic, orderField(w.file.file), w.key, w.file.prev.order))
303
+ if (changes.length === 1) return changes[0]
304
+ const combined = combineChanges(changes)
305
+ if (typeof combined === 'string') throw new Error(combined)
306
+ return combined
307
+ }
308
+
309
+ /** Show `file` as a row or a card: one `look:<d>` change. */
310
+ export function planLookInTopic(
311
+ pubkey: string,
312
+ createdAtMs: number,
313
+ args: { topic: ArrangeHolder; file: ArrangedFile; look: FileLook },
314
+ ): UnsignedEvent {
315
+ return fieldChange(pubkey, stamp(createdAtMs, [args.file]), args.topic, lookField(args.file.file), args.look, args.file.prev.look)
316
+ }
317
+
318
+ /**
319
+ * Undo of Remove: put `file` back on the list, then back in the place it had
320
+ * — just after `after` (`null`: first). Its look stays: a listing never
321
+ * resets it. Publish `relist`, then send `move` with {@link writeArrangement}
322
+ * over the list read again, so a refusal plans the move again rather than
323
+ * leaving the file back but last. The move is stamped after the listing, so
324
+ * its place is not older than it.
325
+ */
326
+ export function planPutBackInTopic(
327
+ pubkey: string,
328
+ createdAtMs: number,
329
+ args: { topic: ArrangeHolder; file: ArrangedFile; after: string | null },
330
+ ): { relist: UnsignedEvent; move: (files: readonly ArrangedFile[], createdAtMs: number) => UnsignedEvent | null } {
331
+ const listedAt = stamp(createdAtMs, [args.file])
332
+ const holder = ADDRESS.exec(args.topic.address)
333
+ if (!holder) throw new Error(`not a file address: ${JSON.stringify(args.topic.address)}`)
334
+ const relist: UnsignedEvent = {
335
+ pubkey,
336
+ created_at: Math.floor(listedAt / 1000),
337
+ kind: KIND_CHANGE,
338
+ tags: [
339
+ ['a', args.topic.address],
340
+ ['O', holder[3]],
341
+ ['field', `lists:${args.file.file}`],
342
+ ['value', 'true'],
343
+ ['h', args.topic.folder],
344
+ ['ts', String(listedAt)],
345
+ ['o', args.file.file],
346
+ ],
347
+ content: '',
348
+ }
349
+ const move = (files: readonly ArrangedFile[], ms: number) => {
350
+ // Back on the list it has no place yet; a list read before the listing landed does not hold it.
351
+ const held = files.find((f) => f.file === args.file.file)
352
+ const back: ArrangedFile = { ...(held ?? args.file), order: undefined, prevAt: Math.max((held ?? args.file).prevAt, listedAt) }
353
+ const rest = files.filter((f) => f.file !== back.file)
354
+ return planMoveInTopic(pubkey, Math.max(ms, listedAt + 1), { topic: args.topic, files: [...rest, back], file: back.file, after: args.after })
355
+ }
356
+ return { relist, move }
357
+ }
358
+
359
+ /** What {@link writeArrangement} is handed. */
360
+ export interface WriteArrangementArgs {
361
+ /** The topic's files as drawn now, from {@link topicList}; read again after a refusal. */
362
+ files: readonly ArrangedFile[]
363
+ read: () => Promise<readonly ArrangedFile[]>
364
+ /** The change for these files, or `null` when there is nothing left to do. */
365
+ plan: (files: readonly ArrangedFile[], createdAtMs: number) => UnsignedEvent | null
366
+ sign: (event: UnsignedEvent) => Promise<SignedEvent>
367
+ publish: (event: SignedEvent) => Promise<{ ok: boolean; reason?: string }>
368
+ now?: () => number
369
+ tries?: number
370
+ }
371
+
372
+ /**
373
+ * Send an arrangement change, and answer a `conflict:` by reading the topic
374
+ * again and planning the change again on what is there now — never by sending
375
+ * the same places again. A place the planner added for a file nobody had
376
+ * placed would otherwise write over a move someone made since; a look or a
377
+ * single move planned again is §6.2's "set it again after the current one".
378
+ */
379
+ export async function writeArrangement(args: WriteArrangementArgs): Promise<{ ok: true; event?: SignedEvent } | { ok: false; refused: string }> {
380
+ const now = args.now ?? Date.now
381
+ const tries = args.tries ?? 3
382
+ let files = args.files
383
+ let last = ''
384
+ for (let attempt = 0; attempt < tries; attempt++) {
385
+ const change = args.plan(files, now())
386
+ if (!change) return { ok: true }
387
+ const signed = await args.sign(change)
388
+ const answer = await args.publish(signed)
389
+ if (answer.ok) return { ok: true, event: signed }
390
+ last = answer.reason ?? 'refused with no reason'
391
+ const conflict = conflictOf(last)
392
+ if (!conflict || conflict.kind === 'other') return { ok: false, refused: last }
393
+ files = await args.read()
394
+ }
395
+ return { ok: false, refused: `still refused after ${tries} tries: ${last}` }
396
+ }
package/src/topics.ts CHANGED
@@ -51,6 +51,8 @@ import {
51
51
  type ProjectionCache,
52
52
  type QueryFn,
53
53
  } from './projection.js'
54
+ import { fieldsSetBy } from './fields.js'
55
+ import { arrangementHeads, arrangementOf, fileOfListField, readsAs, sortArranged, type FileArrangement } from './topicArrangement.js'
54
56
 
55
57
  const KIND_CHANGE = 1851
56
58
  /** The topic (bare file) kind: the only lists this stage reads (Miky, 2026-10-30). */
@@ -252,7 +254,8 @@ export interface FileTopic {
252
254
  */
253
255
  export async function topicsOfFile(query: QueryFn, file: string): Promise<FileTopic[]> {
254
256
  const isTopic = (entry: ListEntry) => entry.file === file && entry.list.startsWith(`${KIND_TOPIC}:`)
255
- const events = await query([topicsOfFileFilter(file)])
257
+ // Paged: `#o` also returns every `order:` and `look:` change on the file (§6.14), one per move.
258
+ const events = await readAll(query, [topicsOfFileFilter(file)])
256
259
  const lists = [...new Set(listEntries(events).filter(isTopic).map((entry) => entry.list))]
257
260
  if (lists.length === 0) return []
258
261
  const roots = await rootsOf(query, lists)
@@ -265,8 +268,8 @@ export async function topicsOfFile(query: QueryFn, file: string): Promise<FileTo
265
268
  .map((entry) => ({ address: entry.list, root: roots.get(entry.list)!, entry }))
266
269
  }
267
270
 
268
- /** A file on a topic's list. */
269
- export interface ListedFile {
271
+ /** A file on a topic's list, with its place and look there (§6.14). */
272
+ export interface ListedFile extends FileArrangement {
270
273
  /** The listed file's id. */
271
274
  file: string
272
275
  /** Its address when exactly one file the reader can read holds the id, else `null`: not drawn, not counted. */
@@ -276,11 +279,25 @@ export interface ListedFile {
276
279
  entry: ListEntry
277
280
  }
278
281
 
282
+ /** How a topic's list is read. */
283
+ export interface ListReadOptions {
284
+ /** The file kinds a listed id resolves over (§6.14 "A listed id resolves"). */
285
+ fileKinds?: readonly number[]
286
+ /**
287
+ * The relay offers the list read (NIP-11 `file_list.newest`): read each
288
+ * topic's fields as their current changes, at most three per file however
289
+ * often its files were moved, not every change ever made (§6.13). Without
290
+ * it, every change by `#O`, which grows with each move.
291
+ */
292
+ listRead?: boolean
293
+ }
294
+
279
295
  /**
280
296
  * A topic's list, by the topic's address: its entries whose newest value
281
297
  * lists the file, filed in the Folder the topic's root names (§6.2; read here,
282
- * never taken from the caller), in no promised order (sort as the design
283
- * says; `entry.event` carries the time), each id resolved over `fileKinds` by
298
+ * never taken from the caller), in the topic's order (§6.14: placed files by
299
+ * key, then the rest oldest listed first), each with its look and the `prev`
300
+ * its next arrangement follows, each id resolved over `fileKinds` by
284
301
  * `#d`. A topic the reader cannot read, or whose root names no Folder, lists
285
302
  * nothing. An id held by none the reader can read, or by two, resolves to
286
303
  * `null`: a consumer draws and counts only the resolved ones (§3.2, the count
@@ -289,7 +306,7 @@ export interface ListedFile {
289
306
  export async function topicList(
290
307
  query: QueryFn,
291
308
  topic: string,
292
- options: { fileKinds?: readonly number[] } = {},
309
+ options: ListReadOptions = {},
293
310
  ): Promise<ListedFile[]> {
294
311
  return (await topicLists(query, [topic], options)).get(topic) ?? []
295
312
  }
@@ -306,7 +323,7 @@ export async function topicList(
306
323
  export async function topicLists(
307
324
  query: QueryFn,
308
325
  topics: readonly string[],
309
- options: { fileKinds?: readonly number[] } = {},
326
+ options: ListReadOptions = {},
310
327
  ): Promise<Map<string, ListedFile[]>> {
311
328
  const asked = [...new Set(topics)]
312
329
  if (asked.length === 0) return new Map()
@@ -318,27 +335,81 @@ async function listsOf(
318
335
  query: QueryFn,
319
336
  asked: readonly string[],
320
337
  rootsRead: Promise<ReadonlyMap<string, SignedEvent>>,
321
- options: { fileKinds?: readonly number[] },
338
+ options: ListReadOptions,
322
339
  ): Promise<Map<string, ListedFile[]>> {
323
340
  const out = new Map<string, ListedFile[]>(asked.map((topic) => [topic, []]))
324
- // One filter per topic in one request, so a relay's cap on an answer cuts no more than it would for that topic alone.
325
- const [roots, events] = await Promise.all([rootsRead, query(asked.map(topicListFilter))])
341
+ // Paged without the list read: every move adds a change to a topic's `#O`, and an unpaged read would lose its oldest entries.
342
+ const [roots, events] = await Promise.all([rootsRead, options.listRead ? newestChanges(query, asked) : readAll(query, asked.map(topicListFilter))])
326
343
  const folderOfList = (list: string) => {
327
344
  const root = out.has(list) ? roots.get(list) : undefined
328
345
  return root ? folderOf(root) : null
329
346
  }
347
+ if (options.listRead) events.push(...(await skippedAgain(query, events, folderOfList)))
330
348
  const entries = listEntries(events, folderOfList)
331
349
  .filter((entry) => entry.listed)
332
350
  .sort((a, b) => byOrder(a.event, b.event))
333
351
  if (entries.length === 0) return out
352
+ const heads = arrangementHeads(events, folderOfList)
334
353
  const resolved = await resolveIds(query, [...new Set(entries.map((entry) => entry.file))], options.fileKinds)
335
354
  for (const entry of entries) {
336
355
  const held = resolved.get(entry.file)
337
- out.get(entry.list)!.push({ file: entry.file, address: held?.address ?? null, folder: held?.folder ?? null, entry })
356
+ const arranged = arrangementOf(heads.get(entry.list), entry.file, entry.event, roots.get(entry.list)!)
357
+ out.get(entry.list)!.push({ file: entry.file, address: held?.address ?? null, folder: held?.folder ?? null, entry, ...arranged })
338
358
  }
359
+ for (const [list, files] of out) out.set(list, sortArranged(files))
339
360
  return out
340
361
  }
341
362
 
363
+ /**
364
+ * Each topic's fields as their current changes, by the list read (§6.13):
365
+ * one filter per topic, all in one request, so a relay's cap on an answer
366
+ * cuts no more than it would for that topic alone, as the `#O` read does.
367
+ */
368
+ async function newestChanges(query: QueryFn, topics: readonly string[]): Promise<SignedEvent[]> {
369
+ const filters = topics.flatMap((topic) => {
370
+ const [, kind, , d] = ADDRESS.exec(topic) ?? []
371
+ return d ? [{ fields: 'newest', kinds: [Number(kind)], '#d': [d] }] : []
372
+ })
373
+ return filters.length ? query(filters) : []
374
+ }
375
+
376
+ /** The most files per topic the list read asks about again. */
377
+ const SKIPPED_READ_CAP = 100
378
+
379
+ /**
380
+ * The list read answers each field's newest change, a malformed one too (a
381
+ * wrong `o` the relay takes), so the good change before it is not in the
382
+ * answer. For each file where that happened, read its changes on that topic
383
+ * again (`#O` and `#o`), so it folds as a whole `#O` read would.
384
+ */
385
+ async function skippedAgain(
386
+ query: QueryFn,
387
+ events: readonly SignedEvent[],
388
+ folderOfList: (list: string) => string | null | undefined,
389
+ ): Promise<SignedEvent[]> {
390
+ const skipped = new Map<string, Set<string>>()
391
+ for (const event of events) {
392
+ if (event.kind !== KIND_CHANGE) continue
393
+ const list = tagOf(event, 'a')
394
+ // Only a list asked for and readable: a change on another author's same-id topic folds nowhere anyway.
395
+ if (!list || !folderOfList(list)) continue
396
+ for (const { field } of fieldsSetBy(event)) {
397
+ const file = fileOfListField(field)
398
+ // A list entry is read only in the one-field form (§6.14): one set in a `set` is skipped too.
399
+ const entryInSet = field.startsWith('lists:') && tagOf(event, 'field') === undefined
400
+ if (!file || (!entryInSet && readsAs(event, field, list, folderOfList(list)))) continue
401
+ const holder = ADDRESS.exec(list)
402
+ if (!holder) continue
403
+ const files = skipped.get(holder[3]) ?? new Set()
404
+ // Bounded: a change setting hundreds of junk fields asks again about the first few only.
405
+ if (files.size < SKIPPED_READ_CAP) files.add(file)
406
+ skipped.set(holder[3], files)
407
+ }
408
+ }
409
+ if (skipped.size === 0) return []
410
+ return readAll(query, [...skipped].map(([topic, files]) => ({ kinds: [KIND_CHANGE], '#O': [topic], '#o': [...files] })))
411
+ }
412
+
342
413
  /** A topic as {@link readTopics} reads it. */
343
414
  export interface ReadTopic {
344
415
  address: string
@@ -501,7 +572,7 @@ export async function readAll(query: QueryFn, filters: readonly Record<string, u
501
572
  export async function readTopics(
502
573
  query: QueryFn,
503
574
  me: string,
504
- options: { cache?: ProjectionCache; fileKinds?: readonly number[]; nowSeconds?: number; parentsListRead?: boolean } = {},
575
+ options: ListReadOptions & { cache?: ProjectionCache; nowSeconds?: number; parentsListRead?: boolean } = {},
505
576
  ): Promise<TopicsRead> {
506
577
  const nowSeconds = options.nowSeconds ?? Math.floor(Date.now() / 1000)
507
578
  const rootEvents = await readAll(query, [{ kinds: [KIND_TOPIC] }])
@@ -517,7 +588,8 @@ export async function readTopics(
517
588
  if (addresses.length === 0) return { me, topics: [], membership: foldMemberships(me, [], nowSeconds), mentionedIn: new Set(), inConversations: new Map() }
518
589
  const [objects, lists, part, states] = await Promise.all([
519
590
  resolveForeignRoots(addresses, query, options.cache, { archivedWith: true, parentsListRead: options.parentsListRead }),
520
- listsOf(query, addresses, Promise.resolve(roots), options),
591
+ // The relay that offers the list read for the parents offers it for the topics' own lists.
592
+ listsOf(query, addresses, Promise.resolve(roots), { ...options, listRead: options.listRead ?? options.parentsListRead }),
521
593
  // Held once each, though one comment can answer two of the filters: the folds count events.
522
594
  readAll(query, personInFilesFilters(me, addresses)),
523
595
  // The team lists naming a topic, for which of them are spaces.