@agentshouse/mdruntime 0.1.0-ce15a88ae8478 → 0.1.0-ce5c5bb29be5a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/README.md +10 -8
  2. package/dist/adapter.d.ts +30 -34
  3. package/dist/adapter.d.ts.map +1 -1
  4. package/dist/attached-model.d.ts +2 -1
  5. package/dist/attached-model.d.ts.map +1 -1
  6. package/dist/attached-model.js +28 -4
  7. package/dist/binary.d.ts +0 -5
  8. package/dist/binary.d.ts.map +1 -1
  9. package/dist/binary.js +1 -1
  10. package/dist/bootstrap.d.ts +16 -8
  11. package/dist/bootstrap.d.ts.map +1 -1
  12. package/dist/bootstrap.js +139 -94
  13. package/dist/client.d.ts +6 -7
  14. package/dist/client.d.ts.map +1 -1
  15. package/dist/client.js +54 -7
  16. package/dist/collection.d.ts +5 -3
  17. package/dist/collection.d.ts.map +1 -1
  18. package/dist/collection.js +24 -19
  19. package/dist/compile.d.ts.map +1 -1
  20. package/dist/compile.js +82 -30
  21. package/dist/connection.d.ts +8 -7
  22. package/dist/connection.d.ts.map +1 -1
  23. package/dist/connection.js +1047 -915
  24. package/dist/conversation.d.ts +7 -5
  25. package/dist/conversation.d.ts.map +1 -1
  26. package/dist/conversation.js +166 -41
  27. package/dist/durability.d.ts +1 -5
  28. package/dist/durability.d.ts.map +1 -1
  29. package/dist/durability.js +5 -0
  30. package/dist/entry.d.ts +1 -0
  31. package/dist/entry.d.ts.map +1 -1
  32. package/dist/entry.js +1 -1
  33. package/dist/file.d.ts +2 -11
  34. package/dist/file.d.ts.map +1 -1
  35. package/dist/file.js +6 -4
  36. package/dist/fulltext.d.ts.map +1 -1
  37. package/dist/fulltext.js +8 -6
  38. package/dist/host.d.ts +3 -3
  39. package/dist/host.d.ts.map +1 -1
  40. package/dist/host.js +40 -15
  41. package/dist/index.d.ts +9 -9
  42. package/dist/index.d.ts.map +1 -1
  43. package/dist/index.js +4 -3
  44. package/dist/live.d.ts.map +1 -1
  45. package/dist/live.js +2 -1
  46. package/dist/media.d.ts +8 -0
  47. package/dist/media.d.ts.map +1 -0
  48. package/dist/media.js +218 -0
  49. package/dist/memory.d.ts +2 -2
  50. package/dist/memory.d.ts.map +1 -1
  51. package/dist/memory.js +46 -20
  52. package/dist/model.d.ts +6 -1
  53. package/dist/model.d.ts.map +1 -1
  54. package/dist/model.js +7 -10
  55. package/dist/observation.d.ts +1 -0
  56. package/dist/observation.d.ts.map +1 -1
  57. package/dist/observation.js +1 -1
  58. package/dist/outcome.d.ts +13 -6
  59. package/dist/outcome.d.ts.map +1 -1
  60. package/dist/outcome.js +16 -5
  61. package/dist/overlay.js +1 -1
  62. package/dist/paths.d.ts +1 -0
  63. package/dist/paths.d.ts.map +1 -1
  64. package/dist/paths.js +4 -0
  65. package/dist/query.d.ts +6 -0
  66. package/dist/query.d.ts.map +1 -1
  67. package/dist/reach.d.ts +8 -3
  68. package/dist/reach.d.ts.map +1 -1
  69. package/dist/reach.js +69 -46
  70. package/dist/readiness.d.ts +13 -14
  71. package/dist/readiness.d.ts.map +1 -1
  72. package/dist/readiness.js +2 -8
  73. package/dist/recovery.d.ts +1 -0
  74. package/dist/recovery.d.ts.map +1 -1
  75. package/dist/recovery.js +26 -0
  76. package/dist/refresh.d.ts +6 -2
  77. package/dist/refresh.d.ts.map +1 -1
  78. package/dist/refresh.js +61 -27
  79. package/dist/replica.d.ts +5 -5
  80. package/dist/replica.d.ts.map +1 -1
  81. package/dist/replica.js +4 -7
  82. package/dist/room-draft.d.ts +18 -2
  83. package/dist/room-draft.d.ts.map +1 -1
  84. package/dist/room-draft.js +73 -6
  85. package/dist/rows.d.ts +15 -6
  86. package/dist/rows.d.ts.map +1 -1
  87. package/dist/rows.js +102 -74
  88. package/dist/save.d.ts +2 -0
  89. package/dist/save.d.ts.map +1 -1
  90. package/dist/save.js +13 -7
  91. package/dist/shared.d.ts +0 -1
  92. package/dist/shared.d.ts.map +1 -1
  93. package/dist/shared.js +0 -10
  94. package/dist/status.d.ts +4 -3
  95. package/dist/status.d.ts.map +1 -1
  96. package/dist/status.js +3 -1
  97. package/dist/store.d.ts +19 -26
  98. package/dist/store.d.ts.map +1 -1
  99. package/dist/store.js +26 -51
  100. package/dist/stored.d.ts +10 -18
  101. package/dist/stored.d.ts.map +1 -1
  102. package/dist/stored.js +209 -193
  103. package/dist/tables.d.ts +28 -35
  104. package/dist/tables.d.ts.map +1 -1
  105. package/dist/tables.js +58 -50
  106. package/dist/transfer.d.ts +0 -5
  107. package/dist/transfer.d.ts.map +1 -1
  108. package/dist/validation.d.ts +3 -4
  109. package/dist/validation.d.ts.map +1 -1
  110. package/dist/validation.js +35 -96
  111. package/dist/wire.d.ts +5 -5
  112. package/dist/wire.d.ts.map +1 -1
  113. package/dist/wire.js +2 -2
  114. package/dist/words.d.ts.map +1 -1
  115. package/dist/words.js +8 -2
  116. package/package.json +11 -18
  117. package/AGENTS.md +0 -319
  118. package/dist/chat/acts.d.ts +0 -33
  119. package/dist/chat/acts.d.ts.map +0 -1
  120. package/dist/chat/acts.js +0 -36
  121. package/dist/chat/context.d.ts +0 -3
  122. package/dist/chat/context.d.ts.map +0 -1
  123. package/dist/chat/context.js +0 -10
  124. package/dist/chat/earlier.d.ts +0 -10
  125. package/dist/chat/earlier.d.ts.map +0 -1
  126. package/dist/chat/earlier.js +0 -54
  127. package/dist/chat/house.d.ts +0 -25
  128. package/dist/chat/house.d.ts.map +0 -1
  129. package/dist/chat/house.js +0 -38
  130. package/dist/chat/input.d.ts +0 -11
  131. package/dist/chat/input.d.ts.map +0 -1
  132. package/dist/chat/input.js +0 -92
  133. package/dist/chat/kit.d.ts +0 -61
  134. package/dist/chat/kit.d.ts.map +0 -1
  135. package/dist/chat/kit.js +0 -123
  136. package/dist/chat/lasting.d.ts +0 -6
  137. package/dist/chat/lasting.d.ts.map +0 -1
  138. package/dist/chat/lasting.js +0 -19
  139. package/dist/chat/list.d.ts +0 -28
  140. package/dist/chat/list.d.ts.map +0 -1
  141. package/dist/chat/list.js +0 -103
  142. package/dist/chat/markdown.d.ts +0 -21
  143. package/dist/chat/markdown.d.ts.map +0 -1
  144. package/dist/chat/markdown.js +0 -246
  145. package/dist/chat/message.d.ts +0 -7
  146. package/dist/chat/message.d.ts.map +0 -1
  147. package/dist/chat/message.js +0 -254
  148. package/dist/chat/messages.d.ts +0 -35
  149. package/dist/chat/messages.d.ts.map +0 -1
  150. package/dist/chat/messages.js +0 -210
  151. package/dist/chat/name.d.ts +0 -7
  152. package/dist/chat/name.d.ts.map +0 -1
  153. package/dist/chat/name.js +0 -37
  154. package/dist/chat/opened.d.ts +0 -12
  155. package/dist/chat/opened.d.ts.map +0 -1
  156. package/dist/chat/opened.js +0 -19
  157. package/dist/chat/overlay.d.ts +0 -32
  158. package/dist/chat/overlay.d.ts.map +0 -1
  159. package/dist/chat/overlay.js +0 -103
  160. package/dist/chat/parts.d.ts +0 -58
  161. package/dist/chat/parts.d.ts.map +0 -1
  162. package/dist/chat/parts.js +0 -120
  163. package/dist/chat/pieces.d.ts +0 -62
  164. package/dist/chat/pieces.d.ts.map +0 -1
  165. package/dist/chat/pieces.js +0 -1110
  166. package/dist/chat/process.d.ts +0 -20
  167. package/dist/chat/process.d.ts.map +0 -1
  168. package/dist/chat/process.js +0 -32
  169. package/dist/chat/provider.d.ts +0 -70
  170. package/dist/chat/provider.d.ts.map +0 -1
  171. package/dist/chat/provider.js +0 -635
  172. package/dist/chat/question.d.ts +0 -9
  173. package/dist/chat/question.d.ts.map +0 -1
  174. package/dist/chat/question.js +0 -69
  175. package/dist/chat/questions.d.ts +0 -31
  176. package/dist/chat/questions.d.ts.map +0 -1
  177. package/dist/chat/questions.js +0 -111
  178. package/dist/chat/read.d.ts +0 -5
  179. package/dist/chat/read.d.ts.map +0 -1
  180. package/dist/chat/read.js +0 -13
  181. package/dist/chat/ready.d.ts +0 -37
  182. package/dist/chat/ready.d.ts.map +0 -1
  183. package/dist/chat/ready.js +0 -93
  184. package/dist/chat/room-file.d.ts +0 -8
  185. package/dist/chat/room-file.d.ts.map +0 -1
  186. package/dist/chat/room-file.js +0 -72
  187. package/dist/chat/steady.d.ts +0 -3
  188. package/dist/chat/steady.d.ts.map +0 -1
  189. package/dist/chat/steady.js +0 -26
  190. package/dist/chat/style.d.ts +0 -4
  191. package/dist/chat/style.d.ts.map +0 -1
  192. package/dist/chat/style.js +0 -307
  193. package/dist/chat.d.ts +0 -15
  194. package/dist/chat.d.ts.map +0 -1
  195. package/dist/chat.js +0 -8
  196. package/dist/reads.d.ts +0 -11
  197. package/dist/reads.d.ts.map +0 -1
  198. package/dist/reads.js +0 -98
package/dist/stored.js CHANGED
@@ -1,28 +1,37 @@
1
+ import { composedOf } from './collection.js';
1
2
  import { indexDocument, unindexAll, unindexDocuments } from './fulltext.js';
2
- import { derived } from './rows.js';
3
+ import { described, derived } from './rows.js';
3
4
  export const REPLICA_TABLES = [
4
- 'CREATE TABLE replica_keys (key INTEGER PRIMARY KEY AUTOINCREMENT, locator TEXT NOT NULL, transfer TEXT NOT NULL, composition TEXT NOT NULL, rederiving INTEGER)',
5
+ 'CREATE TABLE replica_keys (key INTEGER PRIMARY KEY AUTOINCREMENT, locator TEXT NOT NULL, transfer TEXT NOT NULL, fingerprint TEXT NOT NULL, rederiving INTEGER, searching INTEGER)',
5
6
  'CREATE TABLE replica_rooms (locator TEXT PRIMARY KEY, key INTEGER NOT NULL, record TEXT NOT NULL) WITHOUT ROWID',
6
- 'CREATE TABLE replica_transfers (locator TEXT PRIMARY KEY, key INTEGER NOT NULL, record TEXT NOT NULL) WITHOUT ROWID',
7
7
  'CREATE TABLE replica_entries (id INTEGER PRIMARY KEY, key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, visible INTEGER NOT NULL, present INTEGER NOT NULL, queried INTEGER NOT NULL, binary TEXT, revision TEXT, file_id TEXT, change TEXT, written TEXT, type TEXT, declared TEXT, folder TEXT NOT NULL, fields TEXT, content TEXT)',
8
8
  'CREATE UNIQUE INDEX replica_entries_path ON replica_entries (key, layer, path)',
9
- 'CREATE INDEX replica_entries_visible ON replica_entries (key, visible, path)',
9
+ 'CREATE INDEX replica_entries_visible ON replica_entries (key, visible, present, path, binary)',
10
10
  'CREATE INDEX replica_entries_folder ON replica_entries (key, folder, path)',
11
- 'CREATE TABLE replica_values (doc INTEGER NOT NULL, key INTEGER NOT NULL, form TEXT NOT NULL, held INTEGER NOT NULL, item INTEGER NOT NULL, kind TEXT NOT NULL, value, size INTEGER, single INTEGER NOT NULL, PRIMARY KEY (doc, form, held, item)) WITHOUT ROWID',
12
- 'CREATE INDEX replica_values_lookup ON replica_values (key, form, kind, value)',
13
- 'CREATE TABLE replica_links (doc INTEGER NOT NULL, key INTEGER NOT NULL, kind TEXT NOT NULL, form TEXT NOT NULL, target TEXT NOT NULL, field TEXT NOT NULL, PRIMARY KEY (doc, kind, form, target)) WITHOUT ROWID',
14
- 'CREATE INDEX replica_links_target ON replica_links (key, target, kind, form)',
11
+ 'CREATE INDEX replica_entries_type ON replica_entries (key, type, visible, present, queried, folder)',
12
+ 'CREATE INDEX replica_entries_key ON replica_entries (key)',
13
+ 'CREATE TABLE replica_values (doc INTEGER NOT NULL, keys TEXT NOT NULL, places TEXT NOT NULL, item INTEGER NOT NULL, kind TEXT NOT NULL, value, size INTEGER, PRIMARY KEY (doc, keys, places, item)) WITHOUT ROWID',
14
+ 'CREATE TABLE replica_links (doc INTEGER NOT NULL, key INTEGER NOT NULL, kind TEXT NOT NULL, keys TEXT NOT NULL, places TEXT NOT NULL, target TEXT NOT NULL, field TEXT NOT NULL, PRIMARY KEY (doc, kind, keys, places, target)) WITHOUT ROWID',
15
+ 'CREATE INDEX replica_links_target ON replica_links (key, target, kind, keys)',
15
16
  'CREATE TABLE replica_folders (key INTEGER NOT NULL, layer INTEGER NOT NULL, path TEXT NOT NULL, present INTEGER NOT NULL, PRIMARY KEY (key, layer, path)) WITHOUT ROWID',
16
17
  'CREATE TABLE replica_erasing (key INTEGER PRIMARY KEY, locator TEXT NOT NULL)',
17
- 'CREATE TABLE replica_received (key TEXT PRIMARY KEY, head TEXT NOT NULL) WITHOUT ROWID',
18
- 'CREATE TABLE replica_received_parts (key TEXT NOT NULL, part INTEGER NOT NULL, value TEXT NOT NULL, PRIMARY KEY (key, part)) WITHOUT ROWID',
19
18
  'CREATE TABLE replica_facts (name TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID',
20
19
  'CREATE TABLE replica_presentation (app TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID',
21
20
  ];
22
21
  const CONFIRMED = 0;
23
22
  const OVERLAY = 1;
24
23
  const REFRESHED = 2;
25
- const compositions = new WeakMap();
24
+ const DERIVED_ROWS = 64;
25
+ const ENTRY_WEIGHT = 256;
26
+ const ABSENT = {
27
+ type: null,
28
+ declared: null,
29
+ queried: false,
30
+ folder: '',
31
+ fields: null,
32
+ values: [],
33
+ links: [],
34
+ };
26
35
  function text(value) {
27
36
  return typeof value === 'string' ? value : null;
28
37
  }
@@ -38,84 +47,54 @@ function flag(value) {
38
47
  function paths(list) {
39
48
  return JSON.stringify(list);
40
49
  }
41
- function compositionOf(connection, key) {
42
- let held = compositions.get(connection);
43
- if (held === undefined) {
44
- held = new Map();
45
- compositions.set(connection, held);
46
- }
47
- const [row] = connection.rows('SELECT composition FROM replica_keys WHERE key = ?', [key]);
48
- const textual = text(row?.composition) ?? 'null';
49
- const known = held.get(textual);
50
- if (known !== undefined)
51
- return known;
52
- const parsed = JSON.parse(textual) ?? {
53
- model: { status: 'unavailable', reason: 'not-acquired' },
54
- selection: null,
55
- };
56
- held.clear();
57
- held.set(textual, parsed);
58
- return parsed;
59
- }
60
- export function fingerprint(composition) {
61
- const shape = (model) => model.status === 'unavailable' ? null : model.types;
62
- return JSON.stringify({
63
- model: shape(composition.model),
64
- selection: composition.selection === null
65
- ? null
66
- : { roots: composition.selection.roots, model: shape(composition.selection.model) },
67
- });
68
- }
69
50
  function dropDerived(connection, doc) {
70
51
  connection.run('DELETE FROM replica_values WHERE doc = ?', [doc]);
71
52
  connection.run('DELETE FROM replica_links WHERE doc = ?', [doc]);
72
53
  }
54
+ function scalarText(value) {
55
+ if (value === Infinity)
56
+ return 'Infinity';
57
+ if (value === -Infinity)
58
+ return '-Infinity';
59
+ return JSON.stringify(value);
60
+ }
73
61
  function writeDerived(connection, doc, key, made) {
74
- for (const value of made.values) {
75
- connection.run('INSERT INTO replica_values (doc, key, form, held, item, kind, value, size, single) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', [
62
+ if (made.values.length > 0) {
63
+ const rows = made.values.map((value) => `[${JSON.stringify(value.keys)},${JSON.stringify(value.places)},${value.item},${JSON.stringify(value.kind)},${scalarText(value.value)},${JSON.stringify(value.size)}]`);
64
+ connection.run('INSERT INTO replica_values (doc, keys, places, item, kind, value, size) SELECT ?, value ->> 0, value ->> 1, value ->> 2, value ->> 3, value ->> 4, value ->> 5 FROM json_each(?)', [doc, `[${rows.join(',')}]`]);
65
+ }
66
+ if (made.links.length > 0) {
67
+ connection.run('INSERT OR IGNORE INTO replica_links (doc, key, kind, keys, places, target, field) SELECT ?, ?, value ->> 0, value ->> 1, value ->> 2, value ->> 3, value ->> 4 FROM json_each(?)', [
76
68
  doc,
77
69
  key,
78
- value.form,
79
- value.held,
80
- value.item,
81
- value.kind,
82
- value.value,
83
- value.size,
84
- flag(value.single),
70
+ JSON.stringify(made.links.map((link) => [link.kind, link.keys, link.places, link.target, link.field])),
85
71
  ]);
86
72
  }
87
- for (const link of made.links) {
88
- connection.run('INSERT OR IGNORE INTO replica_links (doc, key, kind, form, target, field) VALUES (?, ?, ?, ?, ?, ?)', [doc, key, link.kind, link.form, link.target, link.field]);
89
- }
90
73
  }
91
74
  function entryId(connection, key, layer, path) {
92
75
  const [row] = connection.rows('SELECT id FROM replica_entries WHERE key = ? AND layer = ? AND path = ?', [key, layer, path]);
93
76
  return row === undefined ? null : number(row.id);
94
77
  }
95
- function removeEntry(connection, key, layer, path) {
78
+ function words() {
79
+ return { dropped: [], made: [] };
80
+ }
81
+ function wordsWritten(connection, owed) {
82
+ unindexDocuments(connection, owed.dropped);
83
+ for (const { id, path, content } of owed.made)
84
+ indexDocument(connection, id, path, content);
85
+ }
86
+ function removeEntry(connection, key, layer, path, owed) {
96
87
  const id = entryId(connection, key, layer, path);
97
88
  if (id === null)
98
89
  return;
99
90
  dropDerived(connection, id);
100
- unindexDocuments(connection, [id]);
91
+ owed.dropped.push(id);
101
92
  connection.run('DELETE FROM replica_entries WHERE id = ?', [id]);
102
93
  }
103
94
  function heldIds(connection, sql, bind) {
104
95
  return connection.rows(sql, bind).map((row) => number(row.id));
105
96
  }
106
- function putEntry(connection, key, layer, entry, present, composition, searched = true) {
107
- removeEntry(connection, key, layer, entry.path);
108
- const made = present
109
- ? derived(entry.path, entry.binary === null ? entry.content : null, composition)
110
- : {
111
- type: null,
112
- declared: null,
113
- queried: false,
114
- folder: '',
115
- fields: null,
116
- values: [],
117
- links: [],
118
- };
97
+ function insertEntry(connection, key, layer, entry, present, made) {
119
98
  connection.run('INSERT INTO replica_entries (key, layer, path, visible, present, queried, content, binary, revision, file_id, change, written, type, declared, folder, fields) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [
120
99
  key,
121
100
  layer,
@@ -134,73 +113,76 @@ function putEntry(connection, key, layer, entry, present, composition, searched
134
113
  made.folder,
135
114
  made.fields,
136
115
  ]);
116
+ }
117
+ function textOf(entry) {
118
+ return entry.binary === null ? entry.content : null;
119
+ }
120
+ function putEntry(connection, key, layer, entry, present, composition, owed, searched = true) {
121
+ removeEntry(connection, key, layer, entry.path, owed);
122
+ const content = textOf(entry);
123
+ const made = present ? derived(entry.path, content, composition) : ABSENT;
124
+ insertEntry(connection, key, layer, entry, present, made);
137
125
  if (!present)
138
126
  return;
139
127
  const id = entryId(connection, key, layer, entry.path);
140
128
  writeDerived(connection, id, key, made);
141
- if (searched && entry.binary === null && entry.content !== null && made.queried) {
142
- indexDocument(connection, id, entry.path, entry.content);
143
- }
129
+ if (searched && content !== null && made.queried)
130
+ owed.made.push({ id, path: entry.path, content });
131
+ }
132
+ function placeEntry(connection, key, entry, owed) {
133
+ removeEntry(connection, key, CONFIRMED, entry.path, owed);
134
+ insertEntry(connection, key, CONFIRMED, entry, true, described(entry.path, textOf(entry)));
144
135
  }
145
136
  function revealed(connection, key, touched) {
146
137
  if (touched.length === 0)
147
138
  return;
148
139
  connection.run(`UPDATE replica_entries SET visible = CASE WHEN layer = ${OVERLAY} THEN present ELSE NOT EXISTS (SELECT 1 FROM replica_entries o WHERE o.key = replica_entries.key AND o.layer = ${OVERLAY} AND o.path = replica_entries.path) END WHERE key = ? AND layer IN (${CONFIRMED}, ${OVERLAY}) AND path IN (SELECT value FROM json_each(?))`, [key, paths(touched)]);
149
140
  }
150
- function newKey(connection, locator, transfer, composition) {
151
- connection.run('INSERT INTO replica_keys (locator, transfer, composition, rederiving) VALUES (?, ?, ?, NULL)', [locator, transfer, JSON.stringify(composition)]);
141
+ function newKey(connection, locator, transfer, fingerprint, built) {
142
+ const owed = built ? null : 0;
143
+ connection.run('INSERT INTO replica_keys (locator, transfer, fingerprint, rederiving, searching) VALUES (?, ?, ?, ?, ?)', [locator, transfer, fingerprint, owed, owed]);
152
144
  const [row] = connection.rows('SELECT last_insert_rowid() AS key');
153
145
  return number(row.key);
154
146
  }
155
- function recomposed(connection, key, composition) {
156
- const [row] = connection.rows('SELECT composition FROM replica_keys WHERE key = ?', [key]);
157
- const before = json(row?.composition);
158
- if (before !== null && fingerprint(before) === fingerprint(composition)) {
159
- connection.run('UPDATE replica_keys SET composition = ? WHERE key = ?', [
160
- JSON.stringify(composition),
161
- key,
162
- ]);
147
+ function recomposed(connection, key, fingerprint) {
148
+ const [row] = connection.rows('SELECT fingerprint FROM replica_keys WHERE key = ?', [key]);
149
+ if (row === undefined || text(row.fingerprint) === fingerprint)
163
150
  return;
164
- }
165
- connection.run('UPDATE replica_keys SET composition = ?, rederiving = 0 WHERE key = ?', [
166
- JSON.stringify(composition),
167
- key,
168
- ]);
151
+ const held = connection.rows('SELECT 1 FROM replica_entries WHERE key = ? LIMIT 1', [key]);
152
+ connection.run(`UPDATE replica_keys SET fingerprint = ?${held.length === 0 ? '' : ', rederiving = 0'} WHERE key = ?`, [fingerprint, key]);
169
153
  }
170
154
  export function entered(connection, locator, composition, rows) {
171
- const key = newKey(connection, locator, '', composition);
172
- for (const entry of rows)
173
- putEntry(connection, key, CONFIRMED, entry, true, composition, false);
155
+ const key = newKey(connection, locator, '', composedOf(composition).fingerprint, true);
156
+ const owed = words();
157
+ for (const entry of rows) {
158
+ putEntry(connection, key, CONFIRMED, entry, true, composition, owed, false);
159
+ }
160
+ wordsWritten(connection, owed);
174
161
  return key;
175
162
  }
176
163
  export function stage(connection, request) {
177
- const [held] = connection.rows('SELECT key FROM replica_transfers WHERE locator = ?', [
178
- request.locator,
179
- ]);
180
164
  const key = request.key ??
181
- (held === undefined
182
- ? newKey(connection, request.locator, request.transfer, request.composition)
183
- : number(held.key));
184
- recomposed(connection, key, request.composition);
185
- if (request.record !== null) {
186
- connection.run('INSERT OR REPLACE INTO replica_transfers (locator, key, record) VALUES (?, ?, ?)', [request.locator, key, request.record]);
187
- }
188
- const composition = compositionOf(connection, key);
189
- for (const entry of request.entries) {
190
- putEntry(connection, key, CONFIRMED, entry, true, composition);
191
- }
165
+ newKey(connection, request.locator, request.transfer, request.fingerprint, false);
166
+ recomposed(connection, key, request.fingerprint);
167
+ connection.run('UPDATE replica_keys SET rederiving = coalesce(rederiving, last), searching = coalesce(searching, last) FROM (SELECT coalesce(max(id), 0) AS last FROM replica_entries) WHERE key = ?', [key]);
168
+ const owed = words();
169
+ for (const entry of request.entries)
170
+ placeEntry(connection, key, entry, owed);
192
171
  for (const path of request.folders) {
193
172
  connection.run('INSERT OR REPLACE INTO replica_folders (key, layer, path, present) VALUES (?, ?, ?, 1)', [key, CONFIRMED, path]);
194
173
  }
174
+ wordsWritten(connection, owed);
195
175
  return key;
196
176
  }
197
177
  export function refreshing(connection, request) {
198
- const held = connection.rows('SELECT 1 FROM replica_rooms WHERE key = ? UNION ALL SELECT 1 FROM replica_transfers WHERE key = ?', [request.key, request.key]);
178
+ const held = connection.rows('SELECT 1 FROM replica_keys k WHERE k.key = ? AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)', [request.key]);
199
179
  if (held.length === 0)
200
180
  return null;
201
- const composition = compositionOf(connection, request.key);
181
+ recomposed(connection, request.key, request.composed.fingerprint);
182
+ const composition = request.composed.composition;
183
+ const owed = words();
202
184
  for (const entry of request.entries) {
203
- putEntry(connection, request.key, REFRESHED, entry, true, composition);
185
+ putEntry(connection, request.key, REFRESHED, entry, true, composition, owed);
204
186
  }
205
187
  for (const path of request.removals) {
206
188
  putEntry(connection, request.key, REFRESHED, {
@@ -211,7 +193,7 @@ export function refreshing(connection, request) {
211
193
  fileId: null,
212
194
  change: null,
213
195
  written: null,
214
- }, false, composition);
196
+ }, false, composition, owed);
215
197
  }
216
198
  for (const [list, present] of [
217
199
  [request.folders, 1],
@@ -221,6 +203,7 @@ export function refreshing(connection, request) {
221
203
  connection.run('INSERT OR REPLACE INTO replica_folders (key, layer, path, present) VALUES (?, ?, ?, ?)', [request.key, REFRESHED, path, present]);
222
204
  }
223
205
  }
206
+ wordsWritten(connection, owed);
224
207
  return null;
225
208
  }
226
209
  function dropLayer(connection, key, layer) {
@@ -262,12 +245,11 @@ function applyRefresh(connection, key) {
262
245
  connection.run('DELETE FROM replica_folders WHERE key = ? AND layer = ?', [key, REFRESHED]);
263
246
  return touched;
264
247
  }
248
+ function beneathRoots(path) {
249
+ return `EXISTS (SELECT 1 FROM json_each(?) r WHERE ${path} = r.value OR (${path} >= r.value || '/' AND ${path} < r.value || '0'))`;
250
+ }
265
251
  function counted(connection, key, roots) {
266
- const outside = roots.map(() => '(path = ? OR (path >= ? AND path < ?))').join(' OR ');
267
- const bind = [key];
268
- for (const root of roots)
269
- bind.push(root, `${root}/`, `${root}0`);
270
- const [row] = connection.rows(`SELECT count(*) AS held FROM replica_entries WHERE key = ? AND visible = 1 AND present = 1 AND (binary IS NOT NULL OR substr(path, -3) = '.md')${roots.length === 0 ? '' : ` AND NOT (${outside})`}`, bind);
252
+ const [row] = connection.rows(`SELECT count(*) AS held FROM replica_entries WHERE key = ? AND visible = 1 AND present = 1 AND (binary IS NOT NULL OR substr(path, -3) = '.md') AND NOT ${beneathRoots('replica_entries.path')}`, [key, JSON.stringify(roots)]);
271
253
  return number(row.held);
272
254
  }
273
255
  export function counts(connection, rooms) {
@@ -281,33 +263,32 @@ function erasing(connection, key, locator) {
281
263
  }
282
264
  function commitRoom(connection, room) {
283
265
  const touched = new Set();
284
- if (room.composition !== undefined)
285
- recomposed(connection, room.key, room.composition);
286
- const composition = compositionOf(connection, room.key);
287
- if (room.published !== undefined) {
288
- if (room.published.from !== null && room.published.from !== room.key) {
289
- erasing(connection, room.published.from, room.locator);
290
- }
291
- connection.run('DELETE FROM replica_transfers WHERE locator = ?', [room.locator]);
266
+ recomposed(connection, room.key, room.composed.fingerprint);
267
+ const composition = room.composed.composition;
268
+ if (room.published !== undefined &&
269
+ room.published.from !== null &&
270
+ room.published.from !== room.key) {
271
+ erasing(connection, room.published.from, room.locator);
292
272
  }
293
273
  if (room.refreshed === true) {
294
274
  for (const path of applyRefresh(connection, room.key))
295
275
  touched.add(path);
296
276
  }
277
+ const owed = words();
297
278
  for (const entry of room.confirmed?.set ?? []) {
298
- putEntry(connection, room.key, CONFIRMED, entry, true, composition);
279
+ putEntry(connection, room.key, CONFIRMED, entry, true, composition, owed);
299
280
  touched.add(entry.path);
300
281
  }
301
282
  for (const path of room.confirmed?.removed ?? []) {
302
- removeEntry(connection, room.key, CONFIRMED, path);
283
+ removeEntry(connection, room.key, CONFIRMED, path, owed);
303
284
  touched.add(path);
304
285
  }
305
286
  for (const entry of room.overlay?.set ?? []) {
306
- putEntry(connection, room.key, OVERLAY, entry, entry.present, composition);
287
+ putEntry(connection, room.key, OVERLAY, entry, entry.present, composition, owed);
307
288
  touched.add(entry.path);
308
289
  }
309
290
  for (const path of room.overlay?.cleared ?? []) {
310
- removeEntry(connection, room.key, OVERLAY, path);
291
+ removeEntry(connection, room.key, OVERLAY, path, owed);
311
292
  touched.add(path);
312
293
  }
313
294
  const changed = [...touched];
@@ -319,21 +300,25 @@ function commitRoom(connection, room) {
319
300
  room.record,
320
301
  ]);
321
302
  }
303
+ wordsWritten(connection, owed);
322
304
  return room.published === undefined ? changed : null;
323
305
  }
324
306
  export function commit(connection, rooms) {
325
307
  const counted = {};
326
308
  const changed = {};
327
- const prepared = {};
309
+ const indexed = {};
328
310
  for (const room of rooms) {
329
311
  changed[room.locator] = commitRoom(connection, room);
330
312
  counted[room.locator] = counts(connection, [
331
313
  { key: room.key, roots: room.roots ?? rootsOf(connection, room.locator) },
332
314
  ])[0];
333
- const [held] = connection.rows('SELECT rederiving FROM replica_keys WHERE key = ?', [room.key]);
334
- prepared[room.locator] = held !== undefined && held.rederiving === null;
315
+ const [held] = connection.rows('SELECT rederiving, searching FROM replica_keys WHERE key = ?', [
316
+ room.key,
317
+ ]);
318
+ indexed[room.locator] =
319
+ held !== undefined && held.rederiving === null && held.searching === null;
335
320
  }
336
- return { counts: counted, changed, prepared };
321
+ return { counts: counted, changed, indexed };
337
322
  }
338
323
  function rootsOf(connection, locator) {
339
324
  const [row] = connection.rows('SELECT record FROM replica_rooms WHERE locator = ?', [locator]);
@@ -342,27 +327,24 @@ function rootsOf(connection, locator) {
342
327
  }
343
328
  export function fenceRooms(connection, locators) {
344
329
  for (const locator of locators) {
345
- for (const row of connection.rows('SELECT key FROM replica_keys WHERE locator = ? UNION SELECT key FROM replica_rooms WHERE locator = ? UNION SELECT key FROM replica_transfers WHERE locator = ?', [locator, locator, locator])) {
330
+ for (const row of connection.rows('SELECT key FROM replica_keys WHERE locator = ? UNION SELECT key FROM replica_rooms WHERE locator = ?', [locator, locator])) {
346
331
  erasing(connection, number(row.key), locator);
347
332
  }
348
333
  connection.run('DELETE FROM replica_rooms WHERE locator = ?', [locator]);
349
- connection.run('DELETE FROM replica_transfers WHERE locator = ?', [locator]);
350
334
  }
351
335
  return null;
352
336
  }
353
- export function abandon(connection, locator, kept) {
354
- const [held] = connection.rows('SELECT key FROM replica_transfers WHERE locator = ?', [locator]);
355
- connection.run('DELETE FROM replica_transfers WHERE locator = ?', [locator]);
356
- if (held === undefined)
357
- return null;
358
- if (!kept)
359
- erasing(connection, number(held.key), locator);
360
- return number(held.key);
337
+ export function abandon(connection, locator) {
338
+ for (const row of connection.rows(`SELECT k.key FROM replica_keys k WHERE k.locator = ?
339
+ AND NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
340
+ AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)`, [locator])) {
341
+ erasing(connection, number(row.key), locator);
342
+ }
343
+ return null;
361
344
  }
362
345
  export function release(connection, key) {
363
346
  const [held] = connection.rows(`SELECT k.locator FROM replica_keys k WHERE k.key = ?
364
- AND NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
365
- AND NOT EXISTS (SELECT 1 FROM replica_transfers t WHERE t.key = k.key)`, [key]);
347
+ AND NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)`, [key]);
366
348
  if (held !== undefined)
367
349
  erasing(connection, key, text(held.locator));
368
350
  return null;
@@ -387,28 +369,70 @@ export function erase(connection, locator, limit) {
387
369
  const [more] = connection.rows(`SELECT 1 FROM replica_erasing${locator === null ? '' : ' WHERE locator = ?'} LIMIT 1`, locator === null ? [] : [locator]);
388
370
  return { remaining: more !== undefined };
389
371
  }
390
- export function rederive(connection, key, limit) {
391
- const [held] = connection.rows('SELECT rederiving FROM replica_keys WHERE key = ?', [key]);
392
- if (held === undefined || held.rederiving === null)
393
- return { done: true };
394
- const after = number(held.rederiving);
395
- const composition = compositionOf(connection, key);
396
- const rows = connection.rows('SELECT id, path, content FROM replica_entries WHERE key = ? AND present = 1 AND id > ? ORDER BY id LIMIT ?', [key, after, limit]);
397
- for (const row of rows) {
398
- const id = number(row.id);
399
- const path = text(row.path);
400
- const made = derived(path, text(row.content), composition);
401
- dropDerived(connection, id);
402
- connection.run('UPDATE replica_entries SET type = ?, declared = ?, queried = ?, fields = ? WHERE id = ?', [made.type, made.declared, flag(made.queried), made.fields, id]);
403
- writeDerived(connection, id, key, made);
372
+ function cursor(value) {
373
+ return value === null || value === undefined ? null : number(value);
374
+ }
375
+ export function rederive(connection, key, composed, bytes, ms, floor) {
376
+ const started = performance.now();
377
+ recomposed(connection, key, composed.fingerprint);
378
+ const [held] = connection.rows('SELECT rederiving, searching FROM replica_keys WHERE key = ?', [
379
+ key,
380
+ ]);
381
+ const deriving = cursor(held?.rederiving);
382
+ const searching = cursor(held?.searching);
383
+ if (deriving === null && searching === null)
384
+ return { done: true, text: 0 };
385
+ const composition = composed.composition;
386
+ let last = Math.min(deriving ?? Infinity, searching ?? Infinity);
387
+ let spent = 0;
388
+ let built = 0;
389
+ let read = false;
390
+ const late = () => spent >= floor && performance.now() - started >= ms;
391
+ const owed = [];
392
+ reading: for (;;) {
393
+ const rows = connection.rows('SELECT id, path, content FROM replica_entries WHERE key = ? AND present = 1 AND id > ? ORDER BY id LIMIT ?', [key, last, DERIVED_ROWS]);
394
+ for (const row of rows) {
395
+ const id = number(row.id);
396
+ const path = text(row.path);
397
+ const content = text(row.content);
398
+ const owedRows = deriving !== null && id > deriving;
399
+ const owedWords = searching !== null && id > searching && content !== null;
400
+ if (owedRows) {
401
+ const made = derived(path, content, composition);
402
+ dropDerived(connection, id);
403
+ writeDerived(connection, id, key, made);
404
+ }
405
+ if (owedWords && described(path, content).queried)
406
+ owed.push({ id, path, content });
407
+ else if (owedRows)
408
+ built += content?.length ?? 0;
409
+ last = id;
410
+ spent += ENTRY_WEIGHT + (content?.length ?? 0);
411
+ if (spent >= bytes || late())
412
+ break reading;
413
+ }
414
+ if (rows.length < DERIVED_ROWS) {
415
+ read = true;
416
+ break;
417
+ }
418
+ }
419
+ let searched = last;
420
+ for (const [at, { id, path, content }] of owed.entries()) {
421
+ indexDocument(connection, id, path, content);
422
+ built += content.length;
423
+ const next = owed[at + 1];
424
+ if (next !== undefined && late()) {
425
+ searched = next.id - 1;
426
+ break;
427
+ }
404
428
  }
405
- const last = rows.at(-1);
406
- const done = rows.length < limit || last === undefined;
407
- connection.run('UPDATE replica_keys SET rederiving = ? WHERE key = ?', [
408
- done ? null : number(last.id),
429
+ const done = read && searched === last;
430
+ connection.run('UPDATE replica_keys SET rederiving = ?, searching = ? WHERE key = ?', [
431
+ deriving === null || read ? null : Math.max(deriving, last),
432
+ searching === null || done ? null : Math.max(searching, searched),
409
433
  key,
410
434
  ]);
411
- return { done };
435
+ return { done, text: built };
412
436
  }
413
437
  function heldEntry(row) {
414
438
  return {
@@ -423,12 +447,31 @@ function heldEntry(row) {
423
447
  };
424
448
  }
425
449
  const ENTRY_COLUMNS = 'e.path, e.content, e.binary, e.revision, e.file_id, e.change, e.written, e.type';
450
+ function unrefreshed(alias) {
451
+ return `NOT EXISTS (SELECT 1 FROM replica_entries r WHERE r.key = ${alias}.key AND r.layer = ${REFRESHED} AND r.path = ${alias}.path)`;
452
+ }
426
453
  function viewed(view, alias) {
427
454
  if (view === 'visible')
428
455
  return `${alias}.visible = 1 AND ${alias}.present = 1`;
429
456
  if (view === 'confirmed')
430
457
  return `${alias}.layer = ${CONFIRMED}`;
431
- return `${alias}.present = 1 AND (${alias}.layer = ${REFRESHED} OR (${alias}.layer = ${CONFIRMED} AND NOT EXISTS (SELECT 1 FROM replica_entries r WHERE r.key = ${alias}.key AND r.layer = ${REFRESHED} AND r.path = ${alias}.path)))`;
458
+ return `${alias}.present = 1 AND (${alias}.layer = ${REFRESHED} OR (${alias}.layer = ${CONFIRMED} AND ${unrefreshed(alias)}))`;
459
+ }
460
+ function layers(view) {
461
+ if (view !== 'refreshed')
462
+ return [viewed(view, 'e')];
463
+ return [
464
+ `e.layer = ${REFRESHED} AND e.present = 1`,
465
+ `e.layer = ${CONFIRMED} AND e.present = 1 AND ${unrefreshed('e')}`,
466
+ ];
467
+ }
468
+ function pathOrdered(connection, columns, key, view, condition, after, limit) {
469
+ const beyond = after === null ? '' : ' AND e.path > ?';
470
+ const parts = layers(view).map((layer) => `SELECT ${columns} FROM replica_entries e WHERE e.key = ? AND ${layer}${condition}${beyond}`);
471
+ return connection.rows(`${parts.join(' UNION ALL ')} ORDER BY path LIMIT ?`, [
472
+ ...parts.flatMap(() => (after === null ? [key] : [key, after])),
473
+ limit,
474
+ ]);
432
475
  }
433
476
  export function entries(connection, key, view, list) {
434
477
  if (list.length === 0)
@@ -459,36 +502,14 @@ export function referrers(connection, key, view, targets, kinds) {
459
502
  kind: text(row.kind),
460
503
  }));
461
504
  }
462
- export function walk(connection, key, view, after, bytes) {
463
- const found = [];
464
- let spent = 0;
465
- let from = after;
466
- for (;;) {
467
- const rows = connection.rows(`SELECT ${ENTRY_COLUMNS} FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')}${from === null ? '' : ' AND e.path > ?'} ORDER BY e.path LIMIT 64`, from === null ? [key] : [key, from]);
468
- for (const row of rows) {
469
- const entry = heldEntry(row);
470
- found.push(entry);
471
- spent += 256 + (entry.content?.length ?? 0) * 2;
472
- if (spent >= bytes)
473
- return { entries: found, next: entry.path };
474
- }
475
- if (rows.length < 64)
476
- return { entries: found, next: null };
477
- from = found.at(-1).path;
478
- }
479
- }
480
505
  export function files(connection, key, view, after, limit) {
481
- return connection
482
- .rows(`SELECT e.path, e.revision FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.content IS NOT NULL AND e.revision IS NOT NULL${after === null ? '' : ' AND e.path > ?'} ORDER BY e.path LIMIT ?`, after === null ? [key, limit] : [key, after, limit])
483
- .map((row) => ({ path: text(row.path), revision: text(row.revision) }));
506
+ return pathOrdered(connection, 'e.path, e.revision', key, view, ' AND e.content IS NOT NULL AND e.revision IS NOT NULL', after, limit).map((row) => ({ path: text(row.path), revision: text(row.revision) }));
507
+ }
508
+ export function folders(connection, key, view, list) {
509
+ return list.filter((path) => connection.rows(`SELECT 1 FROM replica_folders f WHERE f.key = ? AND f.layer = ${CONFIRMED} AND f.present = 1 AND f.path = ? UNION ALL SELECT 1 FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.path >= ? AND e.path < ? LIMIT 1`, [key, path, key, `${path}/`, `${path}0`]).length > 0);
484
510
  }
485
511
  export function typedWithin(connection, key, view, roots, within) {
486
- const inside = roots.map(() => '(e.path = ? OR (e.path >= ? AND e.path < ?))').join(' OR ');
487
- const bind = [key];
488
- for (const root of roots)
489
- bind.push(root, `${root}/`, `${root}0`);
490
- const placed = roots.length === 0 ? (within ? ' AND 0' : '') : ` AND ${within ? '' : 'NOT '}(${inside})`;
491
- return (connection.rows(`SELECT 1 FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.declared IS NOT NULL AND e.declared <> 'any'${placed} LIMIT 1`, bind).length > 0);
512
+ return (connection.rows(`SELECT 1 FROM replica_entries e WHERE e.key = ? AND ${viewed(view, 'e')} AND e.declared IS NOT NULL AND e.declared <> 'any' AND ${within ? '' : 'NOT '}${beneathRoots('e.path')} LIMIT 1`, [key, JSON.stringify(roots)]).length > 0);
492
513
  }
493
514
  export function opened(connection) {
494
515
  const facts = {};
@@ -501,18 +522,14 @@ export function opened(connection) {
501
522
  key: number(row.key),
502
523
  record: text(row.record),
503
524
  })),
504
- transfers: connection.rows('SELECT locator, key, record FROM replica_transfers').map((row) => ({
505
- locator: text(row.locator),
506
- key: number(row.key),
507
- record: text(row.record),
508
- })),
509
525
  keys: connection
510
- .rows('SELECT key, locator, transfer, rederiving FROM replica_keys')
526
+ .rows('SELECT key, locator, transfer, rederiving, searching FROM replica_keys')
511
527
  .map((row) => ({
512
528
  key: number(row.key),
513
529
  locator: text(row.locator),
514
530
  transfer: text(row.transfer),
515
- rederiving: row.rederiving === null ? null : number(row.rederiving),
531
+ rederiving: cursor(row.rederiving),
532
+ searching: cursor(row.searching),
516
533
  })),
517
534
  facts,
518
535
  presentation: connection.rows('SELECT app, value FROM replica_presentation').map((row) => ({
@@ -525,7 +542,6 @@ export function opened(connection) {
525
542
  orphans: connection
526
543
  .rows(`SELECT k.key FROM replica_keys k
527
544
  WHERE NOT EXISTS (SELECT 1 FROM replica_rooms r WHERE r.key = k.key)
528
- AND NOT EXISTS (SELECT 1 FROM replica_transfers t WHERE t.key = k.key)
529
545
  AND NOT EXISTS (SELECT 1 FROM replica_erasing e WHERE e.key = k.key)`)
530
546
  .map((row) => number(row.key)),
531
547
  erasing: connection.rows('SELECT key, locator FROM replica_erasing').map((row) => ({