@qvac/core 1.0.0-rc.2 → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/CHANGELOG.md +158 -2
  2. package/dist-lib/index.d.ts +1 -0
  3. package/dist-lib/index.js +1 -0
  4. package/dist-lib/lib/api/chunk-delta.d.ts +7 -2
  5. package/dist-lib/lib/api/chunk-delta.js +89 -7
  6. package/dist-lib/lib/api/chunks.js +14 -8
  7. package/dist-lib/lib/api/config.d.ts +4 -1
  8. package/dist-lib/lib/api/config.js +45 -4
  9. package/dist-lib/lib/api/custom-models.js +3 -5
  10. package/dist-lib/lib/api/devices.js +3 -1
  11. package/dist-lib/lib/api/files.d.ts +1 -1
  12. package/dist-lib/lib/api/files.js +65 -8
  13. package/dist-lib/lib/api/instrument.js +12 -4
  14. package/dist-lib/lib/api/members.js +7 -13
  15. package/dist-lib/lib/api/mesh.d.ts +1 -3
  16. package/dist-lib/lib/api/mesh.js +2 -2
  17. package/dist-lib/lib/api/notifications.js +8 -4
  18. package/dist-lib/lib/api/projects.js +5 -1
  19. package/dist-lib/lib/api/rag.js +32 -19
  20. package/dist-lib/lib/api/served.js +2 -1
  21. package/dist-lib/lib/api/workspace.js +0 -1
  22. package/dist-lib/lib/assistant/harness.d.ts +8 -2
  23. package/dist-lib/lib/assistant/harness.js +17 -4
  24. package/dist-lib/lib/assistant/history.d.ts +1 -1
  25. package/dist-lib/lib/assistant/history.js +19 -13
  26. package/dist-lib/lib/assistant/message.d.ts +6 -1
  27. package/dist-lib/lib/assistant/message.js +22 -2
  28. package/dist-lib/lib/assistant/model-round.d.ts +5 -0
  29. package/dist-lib/lib/assistant/model-round.js +46 -18
  30. package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
  31. package/dist-lib/lib/assistant/pdf-budget.js +10 -11
  32. package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
  33. package/dist-lib/lib/assistant/pdf-pages.js +75 -0
  34. package/dist-lib/lib/assistant/round-host.js +5 -3
  35. package/dist-lib/lib/assistant-client.d.ts +2 -1
  36. package/dist-lib/lib/assistant-client.js +24 -6
  37. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  38. package/dist-lib/lib/assistant-tree.js +8 -5
  39. package/dist-lib/lib/assistant.d.ts +20 -7
  40. package/dist-lib/lib/assistant.js +468 -240
  41. package/dist-lib/lib/client.js +10 -1
  42. package/dist-lib/lib/compaction.d.ts +3 -1
  43. package/dist-lib/lib/compaction.js +18 -4
  44. package/dist-lib/lib/constants.d.ts +3 -0
  45. package/dist-lib/lib/constants.js +3 -0
  46. package/dist-lib/lib/core/chat-activity.js +13 -16
  47. package/dist-lib/lib/core/chat-copy.js +1 -1
  48. package/dist-lib/lib/core/chat-fork.js +2 -1
  49. package/dist-lib/lib/core/chat-move.d.ts +4 -0
  50. package/dist-lib/lib/core/chat-move.js +42 -11
  51. package/dist-lib/lib/core/chat.d.ts +0 -1
  52. package/dist-lib/lib/core/chat.js +1 -1
  53. package/dist-lib/lib/core/chunk.d.ts +14 -3
  54. package/dist-lib/lib/core/chunk.js +112 -22
  55. package/dist-lib/lib/core/collections.d.ts +12 -1
  56. package/dist-lib/lib/core/collections.js +12 -1
  57. package/dist-lib/lib/core/config-value.d.ts +6 -0
  58. package/dist-lib/lib/core/config-value.js +14 -0
  59. package/dist-lib/lib/core/contract/activity.d.ts +3 -2
  60. package/dist-lib/lib/core/contract/activity.js +39 -21
  61. package/dist-lib/lib/core/contract/agent-join.js +15 -19
  62. package/dist-lib/lib/core/contract/agents.js +8 -7
  63. package/dist-lib/lib/core/contract/chat.d.ts +3 -0
  64. package/dist-lib/lib/core/contract/chat.js +12 -1
  65. package/dist-lib/lib/core/contract/chunks.js +16 -6
  66. package/dist-lib/lib/core/contract/devices.js +6 -4
  67. package/dist-lib/lib/core/contract/index.js +2 -1
  68. package/dist-lib/lib/core/contract/invites.js +12 -31
  69. package/dist-lib/lib/core/contract/joined-meshes.js +2 -1
  70. package/dist-lib/lib/core/contract/mesh-rules.js +4 -0
  71. package/dist-lib/lib/core/contract/model-requests.js +7 -6
  72. package/dist-lib/lib/core/contract/project-device-keys.js +15 -6
  73. package/dist-lib/lib/core/contract/project-host-keys.js +5 -4
  74. package/dist-lib/lib/core/contract/projects.js +12 -13
  75. package/dist-lib/lib/core/contract/public-seeds.js +5 -5
  76. package/dist-lib/lib/core/contract/roles.js +2 -1
  77. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -1
  78. package/dist-lib/lib/core/contract/run-requests.js +19 -13
  79. package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
  80. package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
  81. package/dist-lib/lib/core/contract/workspace.js +4 -10
  82. package/dist-lib/lib/core/contract/writers.js +20 -6
  83. package/dist-lib/lib/core/devices.d.ts +2 -1
  84. package/dist-lib/lib/core/devices.js +4 -2
  85. package/dist-lib/lib/core/document-attachment.d.ts +6 -3
  86. package/dist-lib/lib/core/document-attachment.js +10 -5
  87. package/dist-lib/lib/core/files/attachments.js +3 -3
  88. package/dist-lib/lib/core/files/index.d.ts +8 -5
  89. package/dist-lib/lib/core/files/index.js +109 -18
  90. package/dist-lib/lib/core/files/processing/index.d.ts +1 -1
  91. package/dist-lib/lib/core/files/processing/index.js +8 -2
  92. package/dist-lib/lib/core/files/processing/source.d.ts +1 -0
  93. package/dist-lib/lib/core/files/processing/source.js +3 -0
  94. package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
  95. package/dist-lib/lib/core/files/processing/state.js +7 -0
  96. package/dist-lib/lib/core/files/rag/build.d.ts +1 -0
  97. package/dist-lib/lib/core/files/rag/build.js +14 -1
  98. package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
  99. package/dist-lib/lib/core/files/rag/index.js +1 -1
  100. package/dist-lib/lib/core/files/rag/produce.js +1 -0
  101. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
  102. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +7 -1
  103. package/dist-lib/lib/core/first-event.d.ts +6 -0
  104. package/dist-lib/lib/core/first-event.js +13 -0
  105. package/dist-lib/lib/core/invite.d.ts +2 -2
  106. package/dist-lib/lib/core/invite.js +5 -4
  107. package/dist-lib/lib/core/local/index.d.ts +10 -1
  108. package/dist-lib/lib/core/local/index.js +105 -10
  109. package/dist-lib/lib/core/mesh.d.ts +15 -4
  110. package/dist-lib/lib/core/mesh.js +128 -57
  111. package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
  112. package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
  113. package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
  114. package/dist-lib/lib/core/migrations/file-index.js +6 -2
  115. package/dist-lib/lib/core/migrations/index.d.ts +2 -1
  116. package/dist-lib/lib/core/migrations/index.js +66 -9
  117. package/dist-lib/lib/core/notification.js +96 -23
  118. package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
  119. package/dist-lib/lib/core/pdf/figure.js +64 -0
  120. package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
  121. package/dist-lib/lib/core/pdf/rasterize.js +72 -22
  122. package/dist-lib/lib/core/peer-duplex.js +57 -5
  123. package/dist-lib/lib/core/public-mesh.js +8 -2
  124. package/dist-lib/lib/core/run-requests.js +2 -3
  125. package/dist-lib/lib/core/senders.d.ts +7 -6
  126. package/dist-lib/lib/core/senders.js +23 -30
  127. package/dist-lib/lib/core/shared-agents.d.ts +1 -0
  128. package/dist-lib/lib/core/shared-agents.js +47 -19
  129. package/dist-lib/lib/core/sync/background.d.ts +3 -15
  130. package/dist-lib/lib/core/sync/background.js +25 -102
  131. package/dist-lib/lib/core/sync/devices.js +6 -3
  132. package/dist-lib/lib/core/sync/index.js +12 -2
  133. package/dist-lib/lib/core/sync-status.js +3 -3
  134. package/dist-lib/lib/core.d.ts +59 -10
  135. package/dist-lib/lib/core.js +706 -174
  136. package/dist-lib/lib/credential-pusher.d.ts +11 -7
  137. package/dist-lib/lib/credential-pusher.js +40 -8
  138. package/dist-lib/lib/device-tree.d.ts +18 -0
  139. package/dist-lib/lib/device-tree.js +32 -0
  140. package/dist-lib/lib/doctor/redact.js +1 -1
  141. package/dist-lib/lib/errors.d.ts +4 -0
  142. package/dist-lib/lib/errors.js +4 -0
  143. package/dist-lib/lib/fs-drive-bridge.d.ts +7 -0
  144. package/dist-lib/lib/fs-drive-bridge.js +308 -30
  145. package/dist-lib/lib/integrations/setup-routes.js +7 -5
  146. package/dist-lib/lib/knowledge-search.d.ts +1 -1
  147. package/dist-lib/lib/knowledge-search.js +5 -46
  148. package/dist-lib/lib/kv-cache-retention.d.ts +35 -0
  149. package/dist-lib/lib/kv-cache-retention.js +98 -0
  150. package/dist-lib/lib/latest-frame.d.ts +10 -0
  151. package/dist-lib/lib/latest-frame.js +48 -0
  152. package/dist-lib/lib/model-download.d.ts +6 -1
  153. package/dist-lib/lib/model-download.js +32 -17
  154. package/dist-lib/lib/oauth/resolve.js +13 -5
  155. package/dist-lib/lib/reconnecting-watch.js +19 -11
  156. package/dist-lib/lib/run-notices.d.ts +2 -0
  157. package/dist-lib/lib/run-notices.js +5 -0
  158. package/dist-lib/lib/run-replay.d.ts +13 -1
  159. package/dist-lib/lib/run-replay.js +58 -21
  160. package/dist-lib/schema/build.js +66 -57
  161. package/dist-lib/schema/content.js +2 -1
  162. package/dist-lib/schema/engine.js +37 -5
  163. package/dist-lib/schema/field-order.js +7 -1
  164. package/dist-lib/schema/indexes.d.ts +4 -0
  165. package/dist-lib/schema/indexes.js +18 -0
  166. package/dist-lib/schema/local.js +9 -2
  167. package/dist-lib/schema/qvac.js +112 -4
  168. package/dist-lib/spec/bind.js +3 -0
  169. package/dist-lib/spec/calls.js +1 -0
  170. package/dist-lib/spec/capabilities.d.ts +2 -0
  171. package/dist-lib/spec/hrpc/hrpc.json +13 -0
  172. package/dist-lib/spec/hrpc/index.js +15 -3
  173. package/dist-lib/spec/hrpc/messages.js +1475 -1392
  174. package/dist-lib/spec/hyperschema/index.js +1475 -1392
  175. package/dist-lib/spec/hyperschema/schema.json +86 -1
  176. package/dist-lib/spec/hyperschema/types.d.ts +21 -1
  177. package/dist-lib/spec/local/hyperdb/messages.js +110 -54
  178. package/dist-lib/spec/local/hyperschema/index.js +56 -28
  179. package/dist-lib/spec/local/hyperschema/schema.json +55 -20
  180. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -1
  181. package/dist-lib/spec/mesh/hyperdb/db.json +255 -51
  182. package/dist-lib/spec/mesh/hyperdb/index.js +1063 -404
  183. package/dist-lib/spec/mesh/hyperdb/messages.js +1647 -1214
  184. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +62 -56
  185. package/dist-lib/spec/mesh/hyperdispatch/index.js +156 -139
  186. package/dist-lib/spec/mesh/hyperdispatch/messages.js +787 -629
  187. package/dist-lib/spec/mesh/hyperschema/index.js +787 -629
  188. package/dist-lib/spec/mesh/hyperschema/schema.json +283 -174
  189. package/dist-lib/spec/mesh/hyperschema/types.d.ts +29 -1
  190. package/package.json +23 -17
  191. package/worker/assistant-sidecar.mjs +3 -0
  192. package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
  193. package/dist-lib/lib/core/files/processing/settle.js +0 -86
  194. package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
  195. package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
@@ -5,7 +5,8 @@ import { mustGetActiveAgent } from './agent.js';
5
5
  import { copyAgent } from './agent-copies.js';
6
6
  import { stripped } from './chat-copy.js';
7
7
  import { listsMember, readProject } from './contract/projects.js';
8
- import { ACTIVE_CHATS_BY_AGENT, AGENT_USERS, AGENTS, DEVICES } from './collections.js';
8
+ import { ACTIVE_CHATS_BY_AGENT, AGENT_USERS, AGENTS, AGENTS_BY_PROVIDER, DEVICES } from './collections.js';
9
+ import { devicesOfMember } from './contract/activity.js';
9
10
  export async function ownerOf(workspace, agent) {
10
11
  const device = await workspace.view.get(DEVICES, { id: agent.providerDeviceId });
11
12
  return device?.memberId ?? null;
@@ -133,7 +134,7 @@ export async function pickAgent(scope, { into, agentId, refusal = 'AGENT_NOT_SHA
133
134
  throw refuse(refusal, agentId);
134
135
  }
135
136
  const held = await into.view.get(AGENTS, { id: agentId });
136
- if (await heldByAnother(workspace, held, shared)) {
137
+ if (into.role === 'public' && (await heldByAnother(into, { workspace, held, shared }))) {
137
138
  throw new CoreError('CHAT_AGENT_UNAVAILABLE');
138
139
  }
139
140
  if (refusal === 'CHAT_AGENT_UNAVAILABLE' &&
@@ -164,11 +165,16 @@ export async function pickAgent(scope, { into, agentId, refusal = 'AGENT_NOT_SHA
164
165
  }
165
166
  export async function unpickAgent(workspace, into, agentId) {
166
167
  const copy = await into.view.get(AGENTS, { id: agentId });
167
- if (!picked(copy, workspace) || (await inUse(into, agentId))) {
168
+ if (await inUse(into, agentId)) {
169
+ return;
170
+ }
171
+ if (picked(copy, workspace)) {
172
+ const at = Math.max(Date.now(), copy.updatedAt + 1);
173
+ await into.dispatch('put-agent', { agent: { ...stripped(copy), deletedAt: at, updatedAt: at } });
174
+ }
175
+ else if (into.role !== 'project') {
168
176
  return;
169
177
  }
170
- const at = Math.max(Date.now(), copy.updatedAt + 1);
171
- await into.dispatch('put-agent', { agent: { ...stripped(copy), deletedAt: at, updatedAt: at } });
172
178
  const user = userOf(workspace, into);
173
179
  const userId = user?.memberId ?? user?.projectId;
174
180
  if (user && userId && (await workspace.view.get(AGENT_USERS, { agentId, userId }))) {
@@ -183,7 +189,7 @@ export async function followPickedAgents(workspace, into) {
183
189
  if (!follows(workspace, into, copy)) {
184
190
  continue;
185
191
  }
186
- if (await ownerRemoved(workspace, copy, shared)) {
192
+ if (await ownerRemoved(into, { workspace, copy, shared })) {
187
193
  if (!copy.deletedAt) {
188
194
  const at = copy.updatedAt + 1;
189
195
  await into.dispatch('put-agent', {
@@ -204,7 +210,10 @@ export async function followPickedAgents(workspace, into) {
204
210
  }
205
211
  }
206
212
  }
207
- async function ownerRemoved(workspace, copy, shared) {
213
+ async function ownerRemoved(into, { workspace, copy, shared }) {
214
+ if (into.role === 'project' && shared) {
215
+ return false;
216
+ }
208
217
  const host = await workspace.view.get(DEVICES, { id: copy.providerDeviceId });
209
218
  if (!host?.revoked || !host.memberId) {
210
219
  return false;
@@ -212,32 +221,34 @@ async function ownerRemoved(workspace, copy, shared) {
212
221
  const owner = shared && (await ownerOf(workspace, shared));
213
222
  return !owner || !b4a.equals(owner, host.memberId);
214
223
  }
215
- async function heldByAnother(workspace, held, shared) {
224
+ async function heldByAnother(into, { workspace, held, shared }) {
216
225
  if (!held || b4a.equals(held.providerDeviceId, shared.providerDeviceId)) {
217
226
  return false;
218
227
  }
219
- if (await ownerRemoved(workspace, held, shared)) {
228
+ if (await ownerRemoved(into, { workspace, copy: held, shared })) {
220
229
  return true;
221
230
  }
231
+ if (!held.deletedAt) {
232
+ return false;
233
+ }
222
234
  const [was, now] = [await ownerOf(workspace, held), await ownerOf(workspace, shared)];
223
235
  return !was || !now || !b4a.equals(was, now);
224
236
  }
225
- async function leftProject(into, { workspace, held, shared }) {
237
+ async function leftProject(into, { workspace, shared }) {
226
238
  if (into.role !== 'project') {
227
239
  return false;
228
240
  }
229
- const host = held && (await into.view.get(DEVICES, { id: held.providerDeviceId }));
230
- if (held?.deletedAt && host?.revoked) {
231
- return true;
232
- }
233
- if (await workspace.view.get(AGENT_USERS, { agentId: shared.id, userId: into.key })) {
234
- return false;
235
- }
241
+ const used = await workspace.view.get(AGENT_USERS, { agentId: shared.id, userId: into.key });
242
+ const host = await into.view.get(DEVICES, { id: shared.providerDeviceId });
236
243
  const [owner, project] = [
237
244
  await ownerOf(workspace, shared),
238
245
  await readProject(workspace.view, into.key)
239
246
  ];
240
- return !owner || !project || !listsMember(project, owner);
247
+ const listed = !!owner && !!project && listsMember(project, owner);
248
+ if (host?.revoked) {
249
+ return listed || !used;
250
+ }
251
+ return !used && !listed;
241
252
  }
242
253
  async function unpicked(into, copy, shared) {
243
254
  return !!copy.deletedAt && (!!shared.deletedAt || !(await inUse(into, copy.id)));
@@ -248,8 +259,25 @@ export async function runDefinition(mesh, row) {
248
259
  return row;
249
260
  }
250
261
  const shared = await workspace.view.get(AGENTS, { id: row.id });
251
- if (!shared || shared.deletedAt || (await ownerRemoved(workspace, row, shared))) {
262
+ if (!shared || shared.deletedAt || (await ownerRemoved(mesh, { workspace, copy: row, shared }))) {
252
263
  return null;
253
264
  }
254
265
  return pickedRow(shared, workspace, row.updatedAt);
255
266
  }
267
+ export async function unpickMemberAgents(workspace, into, memberId) {
268
+ for (const device of await devicesOfMember(workspace.view, memberId)) {
269
+ if (!device.memberId || !b4a.equals(device.memberId, memberId)) {
270
+ continue;
271
+ }
272
+ const range = { gte: { providerDeviceId: device.id }, lte: { providerDeviceId: device.id } };
273
+ for (const copy of await into.view.find(AGENTS_BY_PROVIDER, range).toArray()) {
274
+ if (copy.deletedAt || !copy.sharedIn) {
275
+ continue;
276
+ }
277
+ const at = Math.max(Date.now(), copy.updatedAt + 1);
278
+ await into.dispatch('put-agent', {
279
+ agent: { ...stripped(copy), deletedAt: at, updatedAt: at }
280
+ });
281
+ }
282
+ }
283
+ }
@@ -1,8 +1,6 @@
1
1
  import type { AutobeeView } from 'autobee';
2
- import type HyperDB from 'hyperdb';
3
2
  import type { LoggerInterface } from '@qvac/logging';
4
3
  export interface BackgroundSyncHost {
5
- readonly view: HyperDB;
6
4
  readonly autobee: {
7
5
  readonly bee: AutobeeView;
8
6
  };
@@ -10,8 +8,7 @@ export interface BackgroundSyncHost {
10
8
  emit(event: string, ...args: unknown[]): boolean;
11
9
  }
12
10
  export interface BackgroundSyncOptions {
13
- readonly batch: number;
14
- readonly pauseMs: number;
11
+ readonly parallelism?: number;
15
12
  readonly logger: LoggerInterface | null;
16
13
  readonly scope: ReadonlyArray<string>;
17
14
  }
@@ -21,27 +18,18 @@ export interface BackgroundSynced {
21
18
  export declare class BackgroundSync {
22
19
  private readonly _host;
23
20
  private readonly _opts;
24
- private _focus;
25
21
  private _running;
26
22
  private _again;
27
23
  private _stopped;
28
24
  private _paused;
29
25
  private _unpause;
30
- private _wake;
31
- private _entries;
32
26
  constructor(host: BackgroundSyncHost, opts: BackgroundSyncOptions);
33
- focus(chatId: string): void;
34
- run(focus?: string): Promise<boolean>;
27
+ focus(_chatId: string): void;
28
+ run(_focus?: string): Promise<boolean>;
35
29
  pause(): void;
36
30
  resume(): void;
37
31
  stop(): void;
38
32
  private _passes;
39
33
  private _pass;
40
- private _walkChats;
41
- private _next;
42
- private _walkChat;
43
- private _walkTree;
44
- private _step;
45
34
  private get _cut();
46
- private _sleep;
47
35
  }
@@ -1,27 +1,17 @@
1
- import { ACTIVE_CHATS, CHUNKS_BY_CHAT } from '../collections.js';
2
- const CUT_SHORT = new Error('background sync cut short');
3
1
  export class BackgroundSync {
4
2
  _host;
5
3
  _opts;
6
- _focus = null;
7
4
  _running = null;
8
5
  _again = false;
9
6
  _stopped = false;
10
7
  _paused = null;
11
8
  _unpause = noop;
12
- _wake = noop;
13
- _entries = 0;
14
9
  constructor(host, opts) {
15
10
  this._host = host;
16
11
  this._opts = opts;
17
12
  }
18
- focus(chatId) {
19
- this._focus = chatId;
20
- }
21
- run(focus) {
22
- if (focus) {
23
- this._focus = focus;
24
- }
13
+ focus(_chatId) { }
14
+ run(_focus) {
25
15
  if (this._stopped) {
26
16
  return Promise.resolve(false);
27
17
  }
@@ -41,7 +31,6 @@ export class BackgroundSync {
41
31
  }
42
32
  stop() {
43
33
  this._stopped = true;
44
- this._wake();
45
34
  this.resume();
46
35
  }
47
36
  async _passes() {
@@ -59,104 +48,38 @@ export class BackgroundSync {
59
48
  }
60
49
  async _pass() {
61
50
  const { logger, scope } = this._opts;
62
- const started = Date.now();
63
- const head = this._host.autobee.bee.head();
64
- this._entries = 0;
65
- logger?.info(scope, 'background sync started', {
66
- length: head?.length ?? 0,
67
- focus: this._focus !== null
68
- });
69
- let finished = false;
70
- try {
71
- if (head) {
72
- await this._walkChats();
73
- await this._walkTree(head);
74
- }
75
- finished = true;
51
+ while (this._paused) {
52
+ await this._paused;
76
53
  }
77
- catch (err) {
78
- if (err !== CUT_SHORT && !this._cut) {
79
- logger?.warn(scope, 'background sync failed', err);
80
- }
54
+ if (this._cut) {
55
+ return null;
81
56
  }
82
- const ms = Date.now() - started;
83
- logger?.info(scope, 'background sync finished', {
84
- entries: this._entries,
85
- ms,
86
- cutShort: !finished
87
- });
88
- return finished ? (head?.length ?? 0) : null;
89
- }
90
- async _walkChats() {
91
- const snap = this._host.view.snapshot();
57
+ const head = this._host.autobee.bee.head();
58
+ const started = Date.now();
59
+ logger?.info(scope, 'background sync started', { length: head?.length ?? 0 });
92
60
  try {
93
- const walked = new Set();
94
- if (this._focus) {
95
- await this._walkChat(snap, this._focus, walked);
96
- }
97
- const order = [];
98
- for await (const chat of snap.find(ACTIVE_CHATS, { reverse: true })) {
99
- order.push(chat.id);
100
- await this._step();
101
- }
102
- for (let chatId = this._next(order, walked); chatId; chatId = this._next(order, walked)) {
103
- await this._walkChat(snap, chatId, walked);
104
- }
105
- }
106
- finally {
107
- await snap.close();
61
+ await this._host.autobee.bee.download({ parallelism: this._opts.parallelism });
62
+ logger?.info(scope, 'background sync finished', {
63
+ length: head?.length ?? 0,
64
+ ms: Date.now() - started,
65
+ cutShort: false
66
+ });
67
+ return head?.length ?? 0;
108
68
  }
109
- }
110
- _next(order, walked) {
111
- if (this._focus && !walked.has(this._focus)) {
112
- return this._focus;
113
- }
114
- return order.find((chatId) => !walked.has(chatId)) ?? null;
115
- }
116
- async _walkChat(snap, chatId, walked) {
117
- walked.add(chatId);
118
- const range = { gte: { chatId }, lte: { chatId }, reverse: true };
119
- for await (const _chunk of snap.find(CHUNKS_BY_CHAT, range)) {
120
- await this._step();
121
- }
122
- this._opts.logger?.debug(this._opts.scope, 'background sync walked a chat', { chatId });
123
- }
124
- async _walkTree(head) {
125
- const bee = this._host.autobee.bee.checkout(head);
126
- try {
127
- for await (const _entry of bee.createReadStream()) {
128
- await this._step();
69
+ catch (err) {
70
+ if (this._cut) {
71
+ logger?.info(scope, 'background sync finished', {
72
+ ms: Date.now() - started,
73
+ cutShort: true
74
+ });
75
+ return null;
129
76
  }
130
- }
131
- finally {
132
- await bee.close();
133
- }
134
- }
135
- async _step() {
136
- if (++this._entries % this._opts.batch === 0) {
137
- await this._sleep(this._opts.pauseMs);
138
- }
139
- while (this._paused) {
140
- await this._paused;
141
- }
142
- if (this._cut) {
143
- throw CUT_SHORT;
77
+ logger?.warn(scope, 'background sync failed', err);
78
+ return null;
144
79
  }
145
80
  }
146
81
  get _cut() {
147
82
  return this._stopped || !!this._host.closing;
148
83
  }
149
- _sleep(ms) {
150
- if (ms === 0 || this._stopped) {
151
- return Promise.resolve();
152
- }
153
- return new Promise((resolve) => {
154
- const timer = setTimeout(resolve, ms);
155
- this._wake = () => {
156
- clearTimeout(timer);
157
- resolve();
158
- };
159
- });
160
- }
161
84
  }
162
85
  function noop() { }
@@ -7,8 +7,11 @@ export function syncOwnMember(mesh, device) {
7
7
  void mesh.refreshOurMember();
8
8
  }
9
9
  export function syncDeviceRevocation(mesh, device) {
10
- if (!isOwnKickedRow(mesh, device))
10
+ if (!device.writerKey || !b4a.equals(device.writerKey, mesh.localWriterKey)) {
11
11
  return;
12
- mesh.log?.info('sync', 'our writer was revoked (kicked)');
13
- mesh.emit('kicked');
12
+ }
13
+ if (isOwnKickedRow(mesh, device)) {
14
+ mesh.log?.info('sync', 'our writer was revoked (kicked)');
15
+ }
16
+ mesh.kickDirty().catch((err) => mesh.log?.warn('sync', 'reading our kick state failed', err));
14
17
  }
@@ -3,7 +3,7 @@ import { FileChanges } from './files.js';
3
3
  import { syncDeviceRevocation, syncOwnMember } from './devices.js';
4
4
  import { syncChatTombstone } from './chats.js';
5
5
  import { syncNotificationChunk } from './chunks.js';
6
- import { AGENT_HOSTS, AGENT_USERS, AGENTS, CHATS, CHAT_GROUPS, CHUNKS, CHUNK_COUNTERS, DEVICES, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, MEMBER_KEYS, MEMBERS, MESH_NAMES, MESH_SEEDS, PROJECT_DEVICE_KEYS, PROJECT_HOST_KEYS, PROJECT_KEYS, PROJECTS, PUBLIC_MESHES, PUBLIC_SEEDS, PURGED_CHATS, SKILL_SECRETS, WORKSPACE } from '../collections.js';
6
+ import { AGENT_HOSTS, AGENT_USERS, AGENTS, CHATS, CHAT_GROUPS, CHUNKS, CHUNK_COUNTERS, DEVICES, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, MEMBER_KEYS, MEMBERS, MESH_NAMES, MESH_RULES, MESH_SEEDS, PROJECT_DEVICE_KEYS, PROJECT_HOST_KEYS, PROJECT_KEYS, PROJECTS, PUBLIC_MESHES, PUBLIC_SEEDS, PURGED_CHATS, SKILL_SECRETS, WORKSPACE } from '../collections.js';
7
7
  export async function syncChanges(mesh, view, from) {
8
8
  if (mesh.closing)
9
9
  return null;
@@ -57,6 +57,8 @@ function newPass() {
57
57
  artifacts: [],
58
58
  orderDirty: new Set(),
59
59
  purged: new Map(),
60
+ notifications: [],
61
+ rulesRaised: false,
60
62
  unattestedDevice: false,
61
63
  membershipMoved: false,
62
64
  agentsDirty: false,
@@ -84,6 +86,11 @@ async function settle(mesh, pass) {
84
86
  if (pass.attachments.length) {
85
87
  await mesh.fileIndex?.refresh(pass.attachments);
86
88
  }
89
+ if (!pass.rulesRaised) {
90
+ for (const chunk of pass.notifications) {
91
+ syncNotificationChunk(mesh, chunk);
92
+ }
93
+ }
87
94
  if (pass.unattestedDevice) {
88
95
  mesh.emit('unattested-device');
89
96
  }
@@ -135,8 +142,11 @@ async function walkChanges(mesh, { snap, from, files, pass }) {
135
142
  else if (changed(change, CHUNK_COUNTERS)) {
136
143
  pass.orderDirty.add(change.value.chatId);
137
144
  }
145
+ else if (change.collection === MESH_RULES) {
146
+ pass.rulesRaised = true;
147
+ }
138
148
  else if (changed(change, CHUNKS)) {
139
- syncNotificationChunk(mesh, change.value);
149
+ pass.notifications.push(change.value);
140
150
  if (change.value.attachment) {
141
151
  pass.attachments.push(change.value);
142
152
  }
@@ -1,6 +1,6 @@
1
1
  import { unref } from '../unref.js';
2
2
  import b4a from 'b4a';
3
- import { DEVICES } from './collections.js';
3
+ import { liveDevices } from './contract/activity.js';
4
4
  const MIN_SYNCING_MS = 1000;
5
5
  export function createSyncActivity(mesh) {
6
6
  const listeners = new Set();
@@ -120,8 +120,8 @@ async function writerDeviceMap(mesh) {
120
120
  const map = new Map();
121
121
  const snap = mesh.view.snapshot();
122
122
  try {
123
- for await (const device of snap.find(DEVICES)) {
124
- if (device.writerKey && !device.revoked) {
123
+ for await (const device of liveDevices(snap)) {
124
+ if (device.writerKey) {
125
125
  map.set(b4a.toHex(device.writerKey), device);
126
126
  }
127
127
  }
@@ -79,8 +79,19 @@ export interface CoreOptions {
79
79
  readonly suspend: () => Promise<void>;
80
80
  readonly resume: () => Promise<void>;
81
81
  };
82
- syncBatch?: number;
83
- syncPauseMs?: number;
82
+ network?: NetworkGate;
83
+ }
84
+ export interface NetworkConnection {
85
+ destroy(): void;
86
+ }
87
+ export interface NetworkGate {
88
+ readonly onConnection?: (conn: NetworkConnection) => void;
89
+ readonly onMeshTopic?: (topic: MeshTopic) => void;
90
+ }
91
+ export interface MeshTopic {
92
+ readonly meshId: string;
93
+ readonly topic: Buffer;
94
+ readonly swarm: Hyperswarm;
84
95
  }
85
96
  export declare const DEVICE_UNKNOWN = "This device has no device details: pass `device` when creating Core";
86
97
  export interface SharedMesh {
@@ -104,7 +115,6 @@ export interface SkillManagementHooks {
104
115
  }
105
116
  export interface CreateMeshOptions {
106
117
  readonly shared?: boolean;
107
- readonly compute?: boolean;
108
118
  }
109
119
  interface MeshInvitePreview {
110
120
  readonly memberInvite: boolean;
@@ -140,6 +150,7 @@ export declare class Core extends ReadyResource {
140
150
  private readonly _log;
141
151
  private readonly _spec;
142
152
  private readonly _bootstrapNodes;
153
+ private readonly _network;
143
154
  private readonly _device;
144
155
  private readonly _app;
145
156
  private _minSupportedVersion;
@@ -155,7 +166,6 @@ export declare class Core extends ReadyResource {
155
166
  private readonly _blindPeerKeys;
156
167
  private readonly _blindPeeringOpts;
157
168
  private readonly _fileGc;
158
- private readonly _syncPacing;
159
169
  private readonly _autoDownloadModels;
160
170
  private _fileGcTimer;
161
171
  private _fileGcRun;
@@ -187,6 +197,7 @@ export declare class Core extends ReadyResource {
187
197
  private readonly _meshTransitionLock;
188
198
  private readonly _opening;
189
199
  private readonly _retiring;
200
+ private readonly _tearingDown;
190
201
  _debugBeforeMeshOpen?: (meshId: string) => void | Promise<void>;
191
202
  _debugOnMeshOpened?: (mesh: Mesh, meshId: string) => void;
192
203
  private _skills;
@@ -229,6 +240,7 @@ export declare class Core extends ReadyResource {
229
240
  private readonly _groupPassesDue;
230
241
  private readonly _purging;
231
242
  private readonly _publicIds;
243
+ private _publicBoots;
232
244
  private readonly _meshNames;
233
245
  private _ownWorkspaceId;
234
246
  private readonly _publicWriters;
@@ -255,8 +267,12 @@ export declare class Core extends ReadyResource {
255
267
  private _minting;
256
268
  private _backfill;
257
269
  private _leaving;
258
- private _kicked;
259
- private _kickedPending;
270
+ private readonly _kickedMeshes;
271
+ private readonly _kickCloses;
272
+ private readonly _retiredMeshes;
273
+ private _retiredAccount;
274
+ private _kickNotice;
275
+ private _leaveChecks;
260
276
  private readonly _peers;
261
277
  private _peerArrivedSuspended;
262
278
  private readonly _statusListeners;
@@ -271,7 +287,7 @@ export declare class Core extends ReadyResource {
271
287
  private _systemInfo;
272
288
  private _statusError;
273
289
  private _upgradeRequired;
274
- constructor(transport: Transport, { storagePath, store, swarm, wakeup, logger, spec, bootstrapNodes, device, app, minSupportedVersion, otaConfigCore, blindPeerKeys, blindPeeringOpts, fileGc, autoDownloadModels, meshSeed, meshKey, additionalNodes, listModels, deleteModel, clearKvCache, deleteChatKvCache, compactChat, modelCatalogWatch, retryModelDownload, transcribe, detectLanguage, previewModelSource, speak, speakStream, embedder, extractors, skills, skillCatalog, tools, hostsModelSource, vectorIndex, apiLifecycle, syncBatch, syncPauseMs }: CoreOptions);
290
+ constructor(transport: Transport, { storagePath, store, swarm, wakeup, logger, spec, bootstrapNodes, device, app, minSupportedVersion, otaConfigCore, blindPeerKeys, blindPeeringOpts, fileGc, autoDownloadModels, meshSeed, meshKey, additionalNodes, listModels, deleteModel, clearKvCache, deleteChatKvCache, compactChat, modelCatalogWatch, retryModelDownload, transcribe, detectLanguage, previewModelSource, speak, speakStream, embedder, extractors, skills, skillCatalog, tools, hostsModelSource, vectorIndex, apiLifecycle, network }: CoreOptions);
275
291
  get devicePublicKey(): Buffer | null;
276
292
  private _requireDeviceKeyPair;
277
293
  get fsDrives(): DriveHost | null;
@@ -283,7 +299,6 @@ export declare class Core extends ReadyResource {
283
299
  _open(): Promise<void>;
284
300
  private _runBootOffline;
285
301
  private _openBootMesh;
286
- private _retireComputeMeshes;
287
302
  private _runBootOnline;
288
303
  private _reopenMesh;
289
304
  get otaConfigCore(): ConfigCore | null;
@@ -334,6 +349,7 @@ export declare class Core extends ReadyResource {
334
349
  private _closeBlindPeering;
335
350
  private _closeRecovery;
336
351
  private _joinBirthMember;
352
+ private _ownBirthMember;
337
353
  private _knownBirthMember;
338
354
  private _recoveryBirthMember;
339
355
  private _holdIdentityPhrase;
@@ -363,13 +379,30 @@ export declare class Core extends ReadyResource {
363
379
  private _retirePooledMesh;
364
380
  private _retirePooled;
365
381
  private _teardownMesh;
382
+ private _tearDown;
366
383
  private _stopMeshServices;
367
384
  private _stopEntryIndexing;
368
385
  private _releaseVectorIndex;
369
386
  private _withdrawFromMesh;
387
+ private _reviewKick;
370
388
  private _onKicked;
389
+ private _kickOutlivesTransition;
390
+ private _onKickVoided;
391
+ private _readKick;
392
+ private _removedAsMember;
393
+ private _followKick;
394
+ private _markJoinedKicked;
395
+ private _closeKicked;
396
+ private _projectRowsOf;
397
+ private _isHidden;
398
+ private _retireKickedAccount;
399
+ private _restoreAccount;
400
+ private _dropOwnMint;
371
401
  private _replaceRevokedAccount;
402
+ private _leaveKicked;
372
403
  private _departKicked;
404
+ private _noteKickedProject;
405
+ private _departProjectsOf;
373
406
  private _departRevokedMesh;
374
407
  private _archiveMembership;
375
408
  private _requestFreshMesh;
@@ -432,6 +465,7 @@ export declare class Core extends ReadyResource {
432
465
  private _onMeshDead;
433
466
  private _onMeshStalled;
434
467
  private _wireMesh;
468
+ private _onMeshesDirty;
435
469
  private _listenMesh;
436
470
  private _wireForeign;
437
471
  private _adoptAccount;
@@ -463,6 +497,7 @@ export declare class Core extends ReadyResource {
463
497
  private _settlePublicMesh;
464
498
  private _publicRecord;
465
499
  private _savePublicRecord;
500
+ private _takePublicBoot;
466
501
  private _openPublicMesh;
467
502
  private _mintPublicBirth;
468
503
  private _mergeLostMint;
@@ -484,6 +519,9 @@ export declare class Core extends ReadyResource {
484
519
  private _retirePublicMeshes;
485
520
  private _retirePublicMesh;
486
521
  private _leavePublicMesh;
522
+ private _dropKickedCopy;
523
+ private _rotatePublicWriters;
524
+ private _restorePublicWriters;
487
525
  private _pooled;
488
526
  private _isOurMemberDevice;
489
527
  private _accountMesh;
@@ -494,6 +532,7 @@ export declare class Core extends ReadyResource {
494
532
  private _fileIndexFor;
495
533
  private _indexMeshFiles;
496
534
  private _syncMeshInBackground;
535
+ private _backfillChatChunks;
497
536
  private _servedFile;
498
537
  clearOldFiles({ olderThanMs, includeOwn }: ClearOldFilesOptions): Promise<number>;
499
538
  private _startFileGc;
@@ -510,7 +549,11 @@ export declare class Core extends ReadyResource {
510
549
  private _recordDeparture;
511
550
  private _departureHasAudience;
512
551
  private _announceDeparture;
513
- leaveMesh(meshId: string): Promise<void>;
552
+ leaveMesh(meshId: string, { workspace }?: {
553
+ workspace?: boolean | undefined;
554
+ }): Promise<void>;
555
+ private _checkLeave;
556
+ private _leaveChecked;
514
557
  leaveWorkspace(meshId: string): Promise<void>;
515
558
  private _leavingMember;
516
559
  private _leaveAsMember;
@@ -536,7 +579,7 @@ export declare class Core extends ReadyResource {
536
579
  setMeshName(meshId: string, name: string): Promise<void>;
537
580
  createWorkspace({ name, icon }?: CreateWorkspaceOptions): Promise<string>;
538
581
  private _defaultWorkspaceName;
539
- createMesh({ shared, compute }?: CreateMeshOptions): Promise<string>;
582
+ createMesh({ shared }?: CreateMeshOptions): Promise<string>;
540
583
  createProject(workspaceId: string, { name }: ProjectIdentity): Promise<string>;
541
584
  private _mintProject;
542
585
  private _listProject;
@@ -569,6 +612,7 @@ export declare class Core extends ReadyResource {
569
612
  waitForWritable(): Promise<void>;
570
613
  private _assertMeshIsHere;
571
614
  getMeshInvite(opts?: MeshInviteOptions): Promise<MeshInvite | null>;
615
+ private _publicHeads;
572
616
  private _delegatePairing;
573
617
  private _shortestAnchorProof;
574
618
  previewMeshInvite(invite: string): MeshInvitePreview;
@@ -576,12 +620,15 @@ export declare class Core extends ReadyResource {
576
620
  pushSkillSecret(request: SkillSecretRequest): Promise<void>;
577
621
  private _followSkillSecrets;
578
622
  private _followJoinedMeshes;
623
+ private _followJoinedMeshesPass;
579
624
  private _followMeshRows;
580
625
  private _trackFollow;
581
626
  private _retireLeftMesh;
582
627
  private _archiveLeftMeshRow;
583
628
  private _openFollowedMesh;
584
629
  private _followProjects;
630
+ private _passProjects;
631
+ private _mayFollowProjects;
585
632
  private _sealProjectKeys;
586
633
  private _sealWorkspaceProjectKeys;
587
634
  private _sealListedMembers;
@@ -667,9 +714,11 @@ export declare class Core extends ReadyResource {
667
714
  private _diagnosticsSource;
668
715
  private _memberDiagnostics;
669
716
  private _meshDiagnostics;
717
+ private _meshProgress;
670
718
  private _diagnostics;
671
719
  private _statusSnapshot;
672
720
  acknowledgeKicked(): void;
721
+ private _accountKicked;
673
722
  private _statusState;
674
723
  private _accountWritable;
675
724
  private _chatSenders;