@qvac/core 1.0.0-rc.2 → 1.0.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/CHANGELOG.md +158 -2
  2. package/dist-lib/index.d.ts +1 -0
  3. package/dist-lib/index.js +1 -0
  4. package/dist-lib/lib/api/chunk-delta.d.ts +7 -2
  5. package/dist-lib/lib/api/chunk-delta.js +89 -7
  6. package/dist-lib/lib/api/chunks.js +14 -8
  7. package/dist-lib/lib/api/config.d.ts +4 -1
  8. package/dist-lib/lib/api/config.js +45 -4
  9. package/dist-lib/lib/api/custom-models.js +3 -5
  10. package/dist-lib/lib/api/devices.js +3 -1
  11. package/dist-lib/lib/api/files.d.ts +1 -1
  12. package/dist-lib/lib/api/files.js +65 -8
  13. package/dist-lib/lib/api/instrument.js +12 -4
  14. package/dist-lib/lib/api/members.js +7 -13
  15. package/dist-lib/lib/api/mesh.d.ts +1 -3
  16. package/dist-lib/lib/api/mesh.js +2 -2
  17. package/dist-lib/lib/api/notifications.js +8 -4
  18. package/dist-lib/lib/api/projects.js +5 -1
  19. package/dist-lib/lib/api/rag.js +32 -19
  20. package/dist-lib/lib/api/served.js +2 -1
  21. package/dist-lib/lib/api/workspace.js +0 -1
  22. package/dist-lib/lib/assistant/harness.d.ts +8 -2
  23. package/dist-lib/lib/assistant/harness.js +17 -4
  24. package/dist-lib/lib/assistant/history.d.ts +1 -1
  25. package/dist-lib/lib/assistant/history.js +19 -13
  26. package/dist-lib/lib/assistant/message.d.ts +6 -1
  27. package/dist-lib/lib/assistant/message.js +22 -2
  28. package/dist-lib/lib/assistant/model-round.d.ts +5 -0
  29. package/dist-lib/lib/assistant/model-round.js +46 -18
  30. package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
  31. package/dist-lib/lib/assistant/pdf-budget.js +10 -11
  32. package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
  33. package/dist-lib/lib/assistant/pdf-pages.js +75 -0
  34. package/dist-lib/lib/assistant/round-host.js +5 -3
  35. package/dist-lib/lib/assistant-client.d.ts +2 -1
  36. package/dist-lib/lib/assistant-client.js +24 -6
  37. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  38. package/dist-lib/lib/assistant-tree.js +8 -5
  39. package/dist-lib/lib/assistant.d.ts +20 -7
  40. package/dist-lib/lib/assistant.js +468 -240
  41. package/dist-lib/lib/client.js +10 -1
  42. package/dist-lib/lib/compaction.d.ts +3 -1
  43. package/dist-lib/lib/compaction.js +18 -4
  44. package/dist-lib/lib/constants.d.ts +3 -0
  45. package/dist-lib/lib/constants.js +3 -0
  46. package/dist-lib/lib/core/chat-activity.js +13 -16
  47. package/dist-lib/lib/core/chat-copy.js +1 -1
  48. package/dist-lib/lib/core/chat-fork.js +2 -1
  49. package/dist-lib/lib/core/chat-move.d.ts +4 -0
  50. package/dist-lib/lib/core/chat-move.js +42 -11
  51. package/dist-lib/lib/core/chat.d.ts +0 -1
  52. package/dist-lib/lib/core/chat.js +1 -1
  53. package/dist-lib/lib/core/chunk.d.ts +14 -3
  54. package/dist-lib/lib/core/chunk.js +112 -22
  55. package/dist-lib/lib/core/collections.d.ts +12 -1
  56. package/dist-lib/lib/core/collections.js +12 -1
  57. package/dist-lib/lib/core/config-value.d.ts +6 -0
  58. package/dist-lib/lib/core/config-value.js +14 -0
  59. package/dist-lib/lib/core/contract/activity.d.ts +3 -2
  60. package/dist-lib/lib/core/contract/activity.js +39 -21
  61. package/dist-lib/lib/core/contract/agent-join.js +15 -19
  62. package/dist-lib/lib/core/contract/agents.js +8 -7
  63. package/dist-lib/lib/core/contract/chat.d.ts +3 -0
  64. package/dist-lib/lib/core/contract/chat.js +12 -1
  65. package/dist-lib/lib/core/contract/chunks.js +16 -6
  66. package/dist-lib/lib/core/contract/devices.js +6 -4
  67. package/dist-lib/lib/core/contract/index.js +2 -1
  68. package/dist-lib/lib/core/contract/invites.js +12 -31
  69. package/dist-lib/lib/core/contract/joined-meshes.js +2 -1
  70. package/dist-lib/lib/core/contract/mesh-rules.js +4 -0
  71. package/dist-lib/lib/core/contract/model-requests.js +7 -6
  72. package/dist-lib/lib/core/contract/project-device-keys.js +15 -6
  73. package/dist-lib/lib/core/contract/project-host-keys.js +5 -4
  74. package/dist-lib/lib/core/contract/projects.js +12 -13
  75. package/dist-lib/lib/core/contract/public-seeds.js +5 -5
  76. package/dist-lib/lib/core/contract/roles.js +2 -1
  77. package/dist-lib/lib/core/contract/run-requests.d.ts +1 -1
  78. package/dist-lib/lib/core/contract/run-requests.js +19 -13
  79. package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
  80. package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
  81. package/dist-lib/lib/core/contract/workspace.js +4 -10
  82. package/dist-lib/lib/core/contract/writers.js +20 -6
  83. package/dist-lib/lib/core/devices.d.ts +2 -1
  84. package/dist-lib/lib/core/devices.js +4 -2
  85. package/dist-lib/lib/core/document-attachment.d.ts +6 -3
  86. package/dist-lib/lib/core/document-attachment.js +10 -5
  87. package/dist-lib/lib/core/files/attachments.js +3 -3
  88. package/dist-lib/lib/core/files/index.d.ts +8 -5
  89. package/dist-lib/lib/core/files/index.js +109 -18
  90. package/dist-lib/lib/core/files/processing/index.d.ts +1 -1
  91. package/dist-lib/lib/core/files/processing/index.js +8 -2
  92. package/dist-lib/lib/core/files/processing/source.d.ts +1 -0
  93. package/dist-lib/lib/core/files/processing/source.js +3 -0
  94. package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
  95. package/dist-lib/lib/core/files/processing/state.js +7 -0
  96. package/dist-lib/lib/core/files/rag/build.d.ts +1 -0
  97. package/dist-lib/lib/core/files/rag/build.js +14 -1
  98. package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
  99. package/dist-lib/lib/core/files/rag/index.js +1 -1
  100. package/dist-lib/lib/core/files/rag/produce.js +1 -0
  101. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
  102. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +7 -1
  103. package/dist-lib/lib/core/first-event.d.ts +6 -0
  104. package/dist-lib/lib/core/first-event.js +13 -0
  105. package/dist-lib/lib/core/invite.d.ts +2 -2
  106. package/dist-lib/lib/core/invite.js +5 -4
  107. package/dist-lib/lib/core/local/index.d.ts +10 -1
  108. package/dist-lib/lib/core/local/index.js +105 -10
  109. package/dist-lib/lib/core/mesh.d.ts +15 -4
  110. package/dist-lib/lib/core/mesh.js +128 -57
  111. package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
  112. package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
  113. package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
  114. package/dist-lib/lib/core/migrations/file-index.js +6 -2
  115. package/dist-lib/lib/core/migrations/index.d.ts +2 -1
  116. package/dist-lib/lib/core/migrations/index.js +66 -9
  117. package/dist-lib/lib/core/notification.js +96 -23
  118. package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
  119. package/dist-lib/lib/core/pdf/figure.js +64 -0
  120. package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
  121. package/dist-lib/lib/core/pdf/rasterize.js +72 -22
  122. package/dist-lib/lib/core/peer-duplex.js +57 -5
  123. package/dist-lib/lib/core/public-mesh.js +8 -2
  124. package/dist-lib/lib/core/run-requests.js +2 -3
  125. package/dist-lib/lib/core/senders.d.ts +7 -6
  126. package/dist-lib/lib/core/senders.js +23 -30
  127. package/dist-lib/lib/core/shared-agents.d.ts +1 -0
  128. package/dist-lib/lib/core/shared-agents.js +47 -19
  129. package/dist-lib/lib/core/sync/background.d.ts +3 -15
  130. package/dist-lib/lib/core/sync/background.js +25 -102
  131. package/dist-lib/lib/core/sync/devices.js +6 -3
  132. package/dist-lib/lib/core/sync/index.js +12 -2
  133. package/dist-lib/lib/core/sync-status.js +3 -3
  134. package/dist-lib/lib/core.d.ts +59 -10
  135. package/dist-lib/lib/core.js +706 -174
  136. package/dist-lib/lib/credential-pusher.d.ts +11 -7
  137. package/dist-lib/lib/credential-pusher.js +40 -8
  138. package/dist-lib/lib/device-tree.d.ts +18 -0
  139. package/dist-lib/lib/device-tree.js +32 -0
  140. package/dist-lib/lib/doctor/redact.js +1 -1
  141. package/dist-lib/lib/errors.d.ts +4 -0
  142. package/dist-lib/lib/errors.js +4 -0
  143. package/dist-lib/lib/fs-drive-bridge.d.ts +7 -0
  144. package/dist-lib/lib/fs-drive-bridge.js +308 -30
  145. package/dist-lib/lib/integrations/setup-routes.js +7 -5
  146. package/dist-lib/lib/knowledge-search.d.ts +1 -1
  147. package/dist-lib/lib/knowledge-search.js +5 -46
  148. package/dist-lib/lib/kv-cache-retention.d.ts +35 -0
  149. package/dist-lib/lib/kv-cache-retention.js +98 -0
  150. package/dist-lib/lib/latest-frame.d.ts +10 -0
  151. package/dist-lib/lib/latest-frame.js +48 -0
  152. package/dist-lib/lib/model-download.d.ts +6 -1
  153. package/dist-lib/lib/model-download.js +32 -17
  154. package/dist-lib/lib/oauth/resolve.js +13 -5
  155. package/dist-lib/lib/reconnecting-watch.js +19 -11
  156. package/dist-lib/lib/run-notices.d.ts +2 -0
  157. package/dist-lib/lib/run-notices.js +5 -0
  158. package/dist-lib/lib/run-replay.d.ts +13 -1
  159. package/dist-lib/lib/run-replay.js +58 -21
  160. package/dist-lib/schema/build.js +66 -57
  161. package/dist-lib/schema/content.js +2 -1
  162. package/dist-lib/schema/engine.js +37 -5
  163. package/dist-lib/schema/field-order.js +7 -1
  164. package/dist-lib/schema/indexes.d.ts +4 -0
  165. package/dist-lib/schema/indexes.js +18 -0
  166. package/dist-lib/schema/local.js +9 -2
  167. package/dist-lib/schema/qvac.js +112 -4
  168. package/dist-lib/spec/bind.js +3 -0
  169. package/dist-lib/spec/calls.js +1 -0
  170. package/dist-lib/spec/capabilities.d.ts +2 -0
  171. package/dist-lib/spec/hrpc/hrpc.json +13 -0
  172. package/dist-lib/spec/hrpc/index.js +15 -3
  173. package/dist-lib/spec/hrpc/messages.js +1475 -1392
  174. package/dist-lib/spec/hyperschema/index.js +1475 -1392
  175. package/dist-lib/spec/hyperschema/schema.json +86 -1
  176. package/dist-lib/spec/hyperschema/types.d.ts +21 -1
  177. package/dist-lib/spec/local/hyperdb/messages.js +110 -54
  178. package/dist-lib/spec/local/hyperschema/index.js +56 -28
  179. package/dist-lib/spec/local/hyperschema/schema.json +55 -20
  180. package/dist-lib/spec/local/hyperschema/types.d.ts +8 -1
  181. package/dist-lib/spec/mesh/hyperdb/db.json +255 -51
  182. package/dist-lib/spec/mesh/hyperdb/index.js +1063 -404
  183. package/dist-lib/spec/mesh/hyperdb/messages.js +1647 -1214
  184. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +62 -56
  185. package/dist-lib/spec/mesh/hyperdispatch/index.js +156 -139
  186. package/dist-lib/spec/mesh/hyperdispatch/messages.js +787 -629
  187. package/dist-lib/spec/mesh/hyperschema/index.js +787 -629
  188. package/dist-lib/spec/mesh/hyperschema/schema.json +283 -174
  189. package/dist-lib/spec/mesh/hyperschema/types.d.ts +29 -1
  190. package/package.json +23 -17
  191. package/worker/assistant-sidecar.mjs +3 -0
  192. package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
  193. package/dist-lib/lib/core/files/processing/settle.js +0 -86
  194. package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
  195. package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
@@ -9,14 +9,14 @@ import HypercoreId from 'hypercore-id-encoding';
9
9
  import ReadyResource from 'ready-resource';
10
10
  import { Files } from './files/index.js';
11
11
  import MeshDefinition from '../../spec/mesh/hyperdb/index.js';
12
- import { Router, encode } from '../../spec/mesh/hyperdispatch/index.js';
12
+ import { Router, encode, version } from '../../spec/mesh/hyperdispatch/index.js';
13
13
  import { syncChanges } from './sync/index.js';
14
14
  import { ViewUpdates } from './sync/view-updates.js';
15
15
  import { BackgroundSync } from './sync/background.js';
16
16
  import { buildDefaultSeed, isOwnKickedRow, isOwnRevokedRow } from './helpers/index.js';
17
17
  import { raceResource } from '../race-resource.js';
18
18
  import { signDeviceAttestation, signWriterClaim, verifyWriterClaim } from './device-attestation.js';
19
- import { devicesByWriterKey, liveDeviceForWriter } from './contract/activity.js';
19
+ import { devicesByWriterKey, devicesOfMember, liveDeviceForWriter } from './contract/activity.js';
20
20
  import { MESH_NAME_ROW } from './contract/mesh-name.js';
21
21
  import { versionedOp } from './contract/oldest-version.js';
22
22
  import { KNOWN_RULES, rulesVersion } from './contract/mesh-rules.js';
@@ -31,10 +31,9 @@ import { CoreError, isCoreError } from '../errors.js';
31
31
  import { ACTIVE_AGENTS, ACTIVE_CHATS, DEVICES, LIVE_WRITERS_BY_MOST_RECENT, MEMBERS, MESH_NAMES } from './collections.js';
32
32
  const [MESH_KEYPAIR, MESH_ENCRYPTION, MESH_DEVICE] = crypto.namespace('qvac-assistant/mesh', 3);
33
33
  const DEFAULT_NAMESPACE = b4a.alloc(32);
34
+ export const SCHEMA_VERSION = version;
34
35
  const SLOW_WRITE_MS = 5000;
35
36
  const SLOW_DRAIN_MS = 15_000;
36
- const SYNC_BATCH = 64;
37
- const SYNC_PAUSE_MS = 100;
38
37
  const BLOB_CORE = 'qvac-blobs';
39
38
  export function meshPublicKeyFromSeed(seed) {
40
39
  return crypto.keyPair(crypto.hash([MESH_KEYPAIR, seed])).publicKey;
@@ -58,7 +57,7 @@ const WARMUP_COLLECTIONS = [
58
57
  '@qvac/members',
59
58
  '@qvac/active-chats'
60
59
  ];
61
- export async function isOurTrustedWriter(snap, writerKey, ourMemberId) {
60
+ export async function isTrustedWriter(snap, writerKey, { ourMemberId }) {
62
61
  const device = await liveDeviceForWriter(snap, writerKey);
63
62
  if (!device) {
64
63
  return false;
@@ -66,7 +65,7 @@ export async function isOurTrustedWriter(snap, writerKey, ourMemberId) {
66
65
  if (!ourMemberId) {
67
66
  return true;
68
67
  }
69
- return !!device.memberId && b4a.equals(device.memberId, ourMemberId);
68
+ return !!device.memberId;
70
69
  }
71
70
  export function claimableDevice(row) {
72
71
  return {
@@ -124,6 +123,8 @@ export class Mesh extends ReadyResource {
124
123
  _slowDrainMs;
125
124
  _stallReported = false;
126
125
  _admitted = false;
126
+ _refused = false;
127
+ _kickSeen = null;
127
128
  _birth = null;
128
129
  _rulesRaised = null;
129
130
  _autobee = null;
@@ -288,8 +289,6 @@ export class Mesh extends ReadyResource {
288
289
  this._scope = ['p2p', 'mesh', b4a.toHex(this._keyPair.publicKey).slice(0, 8)];
289
290
  registerRoutes(this.router, { role: this._role });
290
291
  this._background = new BackgroundSync(this, {
291
- batch: opts.syncBatch ?? SYNC_BATCH,
292
- pauseMs: opts.syncPauseMs ?? SYNC_PAUSE_MS,
293
292
  logger: this._log,
294
293
  scope: this._scope
295
294
  });
@@ -297,6 +296,7 @@ export class Mesh extends ReadyResource {
297
296
  async _open() {
298
297
  this._openedAt = Date.now();
299
298
  this._log?.info(this._scope, 'opening', {
299
+ schema: SCHEMA_VERSION,
300
300
  creator: this._key === null,
301
301
  role: this._role,
302
302
  bootFrom: this._bootFrom?.length ?? null,
@@ -528,25 +528,33 @@ export class Mesh extends ReadyResource {
528
528
  }
529
529
  async _watchStall(pending, watch) {
530
530
  const started = Date.now();
531
- let reported = false;
532
- const timer = setTimeout(() => {
533
- if (this._stallReported) {
534
- return;
535
- }
536
- this._stallReported = true;
537
- reported = true;
538
- this._reportStall(watch, started).catch(noop);
539
- }, watch.ms);
531
+ const clock = { reports: 0, timer: null };
532
+ const report = (wait) => {
533
+ clock.timer = setTimeout(() => {
534
+ if (!this.closing) {
535
+ report(wait * 2);
536
+ }
537
+ if (clock.reports === 0 && this._stallReported) {
538
+ return;
539
+ }
540
+ this._stallReported = true;
541
+ clock.reports++;
542
+ this._reportStall(watch, { started, reports: clock.reports }).catch(noop);
543
+ }, wait);
544
+ };
545
+ report(watch.ms);
540
546
  try {
541
547
  await pending;
542
- this._endStall(watch, { reported, started, failed: false });
548
+ this._endStall(watch, { reported: clock.reports > 0, started, failed: false });
543
549
  }
544
550
  catch (err) {
545
- this._endStall(watch, { reported, started, failed: true, err });
551
+ this._endStall(watch, { reported: clock.reports > 0, started, failed: true, err });
546
552
  throw err;
547
553
  }
548
554
  finally {
549
- clearTimeout(timer);
555
+ if (clock.timer) {
556
+ clearTimeout(clock.timer);
557
+ }
550
558
  }
551
559
  }
552
560
  _endStall({ what, op }, { reported, started, failed, err }) {
@@ -562,23 +570,39 @@ export class Mesh extends ReadyResource {
562
570
  this._log?.info(this._scope, `${what} settled`, { op, ms });
563
571
  }
564
572
  }
565
- async _reportStall({ what, op }, started) {
566
- const { stats, system } = this.autobee;
573
+ async _reportStall({ what, op }, { started, reports }) {
574
+ const { stats, system, writable } = this.autobee;
567
575
  const missing = await this._coresNotHeld([system.bee, this.autobee.bee]).catch(noop);
576
+ const fastForwarding = await this._fastForwardHeld().catch(noop);
577
+ const writers = writable ? undefined : await this._writersHeld().catch(noop);
568
578
  this._log?.warn(this._scope, `${what} still pending`, {
569
579
  op,
580
+ reports,
570
581
  afterMs: Date.now() - started,
571
- writable: this.autobee.writable,
582
+ writable,
572
583
  interrupted: !!this.autobee.interrupted,
573
584
  drains: stats.drains,
574
585
  applies: stats.applies,
575
586
  appends: stats.appends,
576
587
  fastForwards: stats.fastForwards,
588
+ fastForwarding,
577
589
  localLength: this.autobee.local.length,
578
590
  missing,
591
+ writers,
579
592
  blindPeers: this.blindPeerStats()
580
593
  });
581
594
  }
595
+ async _fastForwardHeld() {
596
+ const ff = this.autobee.fastForwarding;
597
+ if (!ff) {
598
+ return null;
599
+ }
600
+ return { to: ff.head.length, ...(await this._coreHeld(ff.head.key)) };
601
+ }
602
+ async _writersHeld() {
603
+ const writers = [...this.autobee.activeWriters];
604
+ return Promise.all(writers.map((writer) => this._coreHeld(writer.core.key)));
605
+ }
582
606
  async _coresNotHeld(views) {
583
607
  const keys = new Map();
584
608
  for (const { context } of views) {
@@ -593,11 +617,14 @@ export class Mesh extends ReadyResource {
593
617
  const core = this._store.get({ key, active: false });
594
618
  try {
595
619
  await core.ready();
620
+ const reach = core.peers.map(remoteReach);
596
621
  return {
597
622
  key: b4a.toHex(key).slice(0, 8),
598
623
  length: core.length,
599
624
  held: core.contiguousLength,
600
- peers: core.peers.length
625
+ peers: core.peers.length,
626
+ peerLength: Math.max(0, ...reach.map((peer) => peer.length)),
627
+ peerHeld: Math.max(0, ...reach.map((peer) => peer.held))
601
628
  };
602
629
  }
603
630
  finally {
@@ -744,17 +771,15 @@ export class Mesh extends ReadyResource {
744
771
  throw this.interrupted;
745
772
  }
746
773
  }
747
- async waitForWritable(timeout) {
774
+ async waitForWritable() {
748
775
  if (this.closing || this.autobee.writable)
749
776
  return;
750
777
  await new Promise((resolve) => {
751
778
  const done = () => {
752
- clearTimeout(timer);
753
779
  this.autobee.off('writable', done);
754
780
  this.off('close', done);
755
781
  resolve();
756
782
  };
757
- const timer = timeout === undefined ? undefined : setTimeout(done, timeout);
758
783
  this.autobee.on('writable', done);
759
784
  this.on('close', done);
760
785
  });
@@ -795,9 +820,9 @@ export class Mesh extends ReadyResource {
795
820
  return false;
796
821
  const snap = reference.db.snapshot();
797
822
  try {
798
- const trusted = await isOurTrustedWriter(snap, key, this._ourMemberId);
823
+ const trusted = await isTrustedWriter(snap, key, this);
799
824
  if (!trusted) {
800
- this._log?.debug(this._scope, 'writer is no device of ours', {
825
+ this._log?.debug(this._scope, 'writer is no live device of this mesh', {
801
826
  writerKey: b4a.toHex(key).slice(0, 16)
802
827
  });
803
828
  }
@@ -815,7 +840,7 @@ export class Mesh extends ReadyResource {
815
840
  for await (const writer of target.db.find(LIVE_WRITERS_BY_MOST_RECENT, {
816
841
  reverse: true
817
842
  })) {
818
- if ((await isOurTrustedWriter(snap, writer.writerKey, this._ourMemberId)) ||
843
+ if ((await isTrustedWriter(snap, writer.writerKey, this)) ||
819
844
  (await this._vouchedByItsRow(target, reference, writer.writerKey))) {
820
845
  this._log?.debug(this._scope, 'vouching for a head', {
821
846
  writerKey: b4a.toHex(writer.writerKey).slice(0, 16),
@@ -824,7 +849,7 @@ export class Mesh extends ReadyResource {
824
849
  return { key: writer.writerKey, length: writer.length };
825
850
  }
826
851
  }
827
- this._log?.debug(this._scope, 'vouching for no head: no writer of ours has been active');
852
+ this._log?.debug(this._scope, 'vouching for no head: no live writer has been active');
828
853
  return null;
829
854
  }
830
855
  finally {
@@ -862,8 +887,11 @@ export class Mesh extends ReadyResource {
862
887
  return { db };
863
888
  }
864
889
  async _autobeeWarmup(view) {
890
+ let snap = null;
865
891
  try {
866
- const counts = await Promise.all(WARMUP_COLLECTIONS.map((collection) => drain(view.db.find(collection))));
892
+ const read = view.db.snapshot();
893
+ snap = read;
894
+ const counts = await Promise.all(WARMUP_COLLECTIONS.map((collection) => drain(read.find(collection))));
867
895
  const warmed = counts.reduce((total, count) => total + count, 0);
868
896
  this._log?.info(this._scope, 'warmed candidate view', {
869
897
  warmed,
@@ -875,6 +903,9 @@ export class Mesh extends ReadyResource {
875
903
  catch (err) {
876
904
  this._log?.warn(this._scope, 'warmup failed', err);
877
905
  }
906
+ finally {
907
+ await snap?.close();
908
+ }
878
909
  }
879
910
  async _autobeeApply(nodes, view, host) {
880
911
  const tx = view.db.transaction();
@@ -983,17 +1014,11 @@ export class Mesh extends ReadyResource {
983
1014
  }
984
1015
  async _readLanding() {
985
1016
  const started = Date.now();
986
- const [own] = await Promise.all([
987
- this.refreshOurMember(),
988
- this.refreshMembership(),
989
- this.refreshName()
990
- ]);
1017
+ await Promise.all([this.refreshOurMember(), this.refreshMembership(), this.refreshName()]);
991
1018
  if (this.closing) {
992
1019
  return;
993
1020
  }
994
- if (own && isOwnKickedRow(this, own)) {
995
- this.emit('kicked');
996
- }
1021
+ await this.kickDirty();
997
1022
  if (this._membership.unattested > 0) {
998
1023
  this.emit('unattested-device');
999
1024
  }
@@ -1005,6 +1030,8 @@ export class Mesh extends ReadyResource {
1005
1030
  async checkOwnAdmission() {
1006
1031
  if (this.autobee.writable) {
1007
1032
  this._admitted = true;
1033
+ this._refused = false;
1034
+ await this.kickDirty();
1008
1035
  return;
1009
1036
  }
1010
1037
  if (!this._admitted || !this._deviceId) {
@@ -1018,33 +1045,68 @@ export class Mesh extends ReadyResource {
1018
1045
  return;
1019
1046
  }
1020
1047
  this._admitted = false;
1021
- this.emit('kicked');
1048
+ this._refused = true;
1049
+ await this.kickDirty();
1050
+ }
1051
+ async kickDirty() {
1052
+ const seen = await this._kickState();
1053
+ if (seen === this._kickSeen || this.closing) {
1054
+ return;
1055
+ }
1056
+ this._kickSeen = seen;
1057
+ this.emit('kick-dirty');
1058
+ }
1059
+ async _kickState() {
1060
+ const writable = this.autobee.writable;
1061
+ const own = await this._ownRow();
1062
+ const member = this.ourMemberId;
1063
+ const devices = !writable && member && this._role === 'workspace'
1064
+ ? await devicesOfMember(this.view, member)
1065
+ : [];
1066
+ return JSON.stringify([
1067
+ writable,
1068
+ this._refused,
1069
+ member && b4a.toHex(member),
1070
+ own && [
1071
+ own.revoked,
1072
+ own.revokedBy && b4a.toHex(own.revokedBy),
1073
+ own.writerKey && b4a.toHex(own.writerKey)
1074
+ ],
1075
+ devices.map((device) => [b4a.toHex(device.id), device.revoked])
1076
+ ]);
1077
+ }
1078
+ async readKicked() {
1079
+ if (this.autobee.writable)
1080
+ return false;
1081
+ const own = await this._ownRow();
1082
+ if (own && isOwnKickedRow(this, own)) {
1083
+ return true;
1084
+ }
1085
+ return this._refused;
1086
+ }
1087
+ async checkResumedRefusal() {
1088
+ if (!(await this.ownWriterRefused())) {
1089
+ return false;
1090
+ }
1091
+ this._refused = true;
1092
+ return true;
1022
1093
  }
1023
1094
  async checkResumedRevocation() {
1024
- const snap = this.view.snapshot();
1025
- try {
1026
- for await (const device of snap.find(DEVICES)) {
1027
- if (!isOwnRevokedRow(this, device))
1028
- continue;
1029
- return isOwnKickedRow(this, device) ? 'kicked' : 'left';
1030
- }
1095
+ const own = await this._ownRow();
1096
+ if (!own || !isOwnRevokedRow(this, own)) {
1031
1097
  return null;
1032
1098
  }
1033
- finally {
1034
- await snap.close();
1035
- }
1099
+ return isOwnKickedRow(this, own) ? 'kicked' : 'left';
1036
1100
  }
1037
1101
  async ownWriterRefused() {
1038
1102
  const info = await this.autobee.system.get(this.localWriterKey, { unflushed: true });
1039
1103
  if (!info?.isRemoved) {
1040
1104
  return false;
1041
1105
  }
1042
- for await (const device of this.view.find(DEVICES)) {
1043
- if (device.writerKey && b4a.equals(device.writerKey, this.localWriterKey)) {
1044
- return false;
1045
- }
1046
- }
1047
- return true;
1106
+ return !(await liveDeviceForWriter(this.view, this.localWriterKey));
1107
+ }
1108
+ async _ownRow() {
1109
+ return this._deviceId ? this.view.get(DEVICES, { id: this._deviceId }) : null;
1048
1110
  }
1049
1111
  }
1050
1112
  export class MeshBatch {
@@ -1082,6 +1144,15 @@ export class MeshBatch {
1082
1144
  }
1083
1145
  }
1084
1146
  function noop() { }
1147
+ function remoteReach(peer) {
1148
+ if (!isRecord(peer)) {
1149
+ return { length: 0, held: 0 };
1150
+ }
1151
+ return {
1152
+ length: typeof peer.remoteLength === 'number' ? peer.remoteLength : 0,
1153
+ held: typeof peer.remoteContiguousLength === 'number' ? peer.remoteContiguousLength : 0
1154
+ };
1155
+ }
1085
1156
  function viewCoreOf(head) {
1086
1157
  return isRecord(head) && b4a.isBuffer(head.key) ? b4a.toHex(head.key).slice(0, 8) : null;
1087
1158
  }
@@ -0,0 +1,4 @@
1
+ import type { Mesh } from '../mesh.ts';
2
+ export declare const REINDEX_FROM_APP_VERSION = "0.21.0";
3
+ export declare function floorExcludesBefore(range: string | null | undefined, release: string): boolean;
4
+ export declare function backfillChatChunks(mesh: Mesh): Promise<number>;
@@ -0,0 +1,47 @@
1
+ import { Range, Version, constants, satisfies } from 'bare-semver';
2
+ import { CHAT_CHUNKS, CHUNK_COUNTERS, PURGED_CHATS } from '../collections.js';
3
+ import { inRole } from '../contract/roles.js';
4
+ import { releaseVersion } from '../min-version.js';
5
+ export const REINDEX_FROM_APP_VERSION = '0.21.0';
6
+ export function floorExcludesBefore(range, release) {
7
+ let floor;
8
+ try {
9
+ floor = Range.parse(range ?? '');
10
+ }
11
+ catch {
12
+ return false;
13
+ }
14
+ const atLeast = Range.parse(`>=${releaseVersion(release)}`);
15
+ return (floor.comparators.length > 0 &&
16
+ floor.comparators.every((set) => set.some((bound) => LOWER_BOUNDS.has(bound.operator) && satisfies(releaseOf(bound.version), atLeast))));
17
+ }
18
+ const LOWER_BOUNDS = new Set([constants.GTE, constants.GT, constants.EQ]);
19
+ function releaseOf({ major, minor, patch }) {
20
+ return new Version(major, minor, patch);
21
+ }
22
+ export async function backfillChatChunks(mesh) {
23
+ if (!inRole(mesh.role, '@qvac/reindex-chat-chunks')) {
24
+ return 0;
25
+ }
26
+ const counters = await mesh.view.find(CHUNK_COUNTERS).toArray();
27
+ const purged = await mesh.view.find(PURGED_CHATS).toArray();
28
+ let sent = 0;
29
+ for (const chat of [...counters, ...purged]) {
30
+ if (mesh.closing || !mesh.writable)
31
+ break;
32
+ const due = 'count' in chat ? !(await holdsEnds(mesh.view, chat)) : await holdsAny(mesh.view, chat);
33
+ if (due) {
34
+ await mesh.dispatch('reindex-chat-chunks', { chatId: chat.chatId });
35
+ sent++;
36
+ }
37
+ }
38
+ return sent;
39
+ }
40
+ async function holdsEnds(db, { chatId, count }) {
41
+ const first = await db.get(CHAT_CHUNKS, { chatId, seq: 0 });
42
+ const last = await db.get(CHAT_CHUNKS, { chatId, seq: count - 1 });
43
+ return first !== null && last !== null;
44
+ }
45
+ async function holdsAny(db, { chatId }) {
46
+ return ((await db.find(CHAT_CHUNKS, { gte: { chatId }, lte: { chatId } }).one()) !== null);
47
+ }
@@ -1,2 +1,2 @@
1
1
  import type { Mesh } from '../mesh.ts';
2
- export declare function backfillFileIndex(mesh: Mesh): Promise<void>;
2
+ export declare function backfillFileIndex(mesh: Mesh): Promise<number>;
@@ -3,26 +3,30 @@ const BATCH = 256;
3
3
  export async function backfillFileIndex(mesh) {
4
4
  const index = mesh.fileIndex;
5
5
  if (!index) {
6
- return;
6
+ return 0;
7
7
  }
8
8
  const snapshot = mesh.view.snapshot();
9
+ let recorded = 0;
9
10
  try {
10
11
  let batch = [];
11
12
  for await (const chunk of attachmentChunks(snapshot)) {
12
13
  if (mesh.closing)
13
- return;
14
+ return recorded;
14
15
  batch.push(chunk);
15
16
  if (batch.length < BATCH) {
16
17
  continue;
17
18
  }
18
19
  await index.record(batch);
20
+ recorded += batch.length;
19
21
  batch = [];
20
22
  }
21
23
  if (batch.length > 0) {
22
24
  await index.record(batch);
25
+ recorded += batch.length;
23
26
  }
24
27
  }
25
28
  finally {
26
29
  await snapshot.close();
27
30
  }
31
+ return recorded;
28
32
  }
@@ -4,5 +4,6 @@ import { type MembershipMigrationContext } from './membership.ts';
4
4
  export type MigrationContext = MembershipMigrationContext & AgentCatalogMigrationContext;
5
5
  export interface RunMigrationsOptions {
6
6
  readonly catalog?: boolean;
7
+ readonly slowMs?: number;
7
8
  }
8
- export declare function runMigrations(mesh: Mesh, context: MigrationContext, { catalog }?: RunMigrationsOptions): Promise<boolean>;
9
+ export declare function runMigrations(mesh: Mesh, context: MigrationContext, { catalog, slowMs }?: RunMigrationsOptions): Promise<boolean>;
@@ -1,17 +1,74 @@
1
+ import b4a from 'b4a';
1
2
  import { inRole } from '../contract/roles.js';
2
3
  import { migrateAgentCatalog } from './agent-catalog.js';
3
4
  import { migrateAgentToolsScope } from './agent-tools-scope.js';
4
5
  import { migrateMembership } from './membership.js';
5
6
  import { sealDeviceRows } from './seal.js';
6
7
  import { reindexAgents } from './agent-indexes.js';
7
- import { reindexDevices } from './device-indexes.js';
8
- export async function runMigrations(mesh, context, { catalog = true } = {}) {
9
- const devices = await reindexDevices(mesh);
10
- const membership = await migrateMembership(mesh, context);
8
+ const SLOW_MIGRATIONS_MS = 15_000;
9
+ export async function runMigrations(mesh, context, { catalog = true, slowMs = SLOW_MIGRATIONS_MS } = {}) {
11
10
  const holdsAgents = mesh.role !== 'public' && inRole(mesh.role, '@qvac/put-agent');
12
- const agents = holdsAgents && catalog && (await migrateAgentCatalog(mesh, context));
13
- const seal = await sealDeviceRows(mesh, context);
14
- const indexed = await reindexAgents(mesh);
15
- const tools = holdsAgents && (await migrateAgentToolsScope(mesh));
16
- return devices || membership || agents || seal || indexed || tools;
11
+ const steps = [
12
+ { name: 'membership', migrate: () => migrateMembership(mesh, context) },
13
+ ...(holdsAgents && catalog
14
+ ? [{ name: 'agent-catalog', migrate: () => migrateAgentCatalog(mesh, context) }]
15
+ : []),
16
+ { name: 'seal', migrate: () => sealDeviceRows(mesh, context) },
17
+ { name: 'reindex-agents', migrate: () => reindexAgents(mesh) },
18
+ ...(holdsAgents
19
+ ? [{ name: 'agent-tools-scope', migrate: () => migrateAgentToolsScope(mesh) }]
20
+ : [])
21
+ ];
22
+ return (await runLogged(mesh, steps, { log: context.log, wait: slowMs })).length > 0;
23
+ }
24
+ async function runLogged(mesh, steps, report) {
25
+ const run = {
26
+ mesh: b4a.toHex(mesh.key).slice(0, 8),
27
+ started: Date.now(),
28
+ step: null,
29
+ timer: null,
30
+ wrote: [],
31
+ ms: {}
32
+ };
33
+ reportWhileSlow(mesh, run, report);
34
+ try {
35
+ await runSteps(steps, run);
36
+ }
37
+ finally {
38
+ if (run.timer) {
39
+ clearTimeout(run.timer);
40
+ }
41
+ report.log?.info(['p2p', 'mesh'], 'migrations ran', {
42
+ mesh: run.mesh,
43
+ ms: Date.now() - run.started,
44
+ wrote: run.wrote,
45
+ steps: run.ms,
46
+ failed: run.step
47
+ });
48
+ }
49
+ return run.wrote;
50
+ }
51
+ async function runSteps(steps, run) {
52
+ for (const { name, migrate } of steps) {
53
+ run.step = name;
54
+ const at = Date.now();
55
+ if (await migrate()) {
56
+ run.wrote.push(name);
57
+ }
58
+ run.ms[name] = Date.now() - at;
59
+ }
60
+ run.step = null;
61
+ }
62
+ function reportWhileSlow(mesh, run, { log, wait }) {
63
+ run.timer = setTimeout(() => {
64
+ if (mesh.closing) {
65
+ return;
66
+ }
67
+ log?.warn(['p2p', 'mesh'], 'migrations still running', {
68
+ mesh: run.mesh,
69
+ step: run.step,
70
+ afterMs: Date.now() - run.started
71
+ });
72
+ reportWhileSlow(mesh, run, { log, wait: wait * 2 });
73
+ }, wait);
17
74
  }