@qvac/core 1.0.0-rc.2 → 1.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -2
- package/dist-lib/index.d.ts +1 -0
- package/dist-lib/index.js +1 -0
- package/dist-lib/lib/api/chunk-delta.d.ts +7 -2
- package/dist-lib/lib/api/chunk-delta.js +89 -7
- package/dist-lib/lib/api/chunks.js +14 -8
- package/dist-lib/lib/api/config.d.ts +4 -1
- package/dist-lib/lib/api/config.js +45 -4
- package/dist-lib/lib/api/custom-models.js +3 -5
- package/dist-lib/lib/api/devices.js +3 -1
- package/dist-lib/lib/api/files.d.ts +1 -1
- package/dist-lib/lib/api/files.js +65 -8
- package/dist-lib/lib/api/instrument.js +12 -4
- package/dist-lib/lib/api/members.js +7 -13
- package/dist-lib/lib/api/mesh.d.ts +1 -3
- package/dist-lib/lib/api/mesh.js +2 -2
- package/dist-lib/lib/api/notifications.js +8 -4
- package/dist-lib/lib/api/projects.js +5 -1
- package/dist-lib/lib/api/rag.js +32 -19
- package/dist-lib/lib/api/served.js +2 -1
- package/dist-lib/lib/api/workspace.js +0 -1
- package/dist-lib/lib/assistant/harness.d.ts +8 -2
- package/dist-lib/lib/assistant/harness.js +17 -4
- package/dist-lib/lib/assistant/history.d.ts +1 -1
- package/dist-lib/lib/assistant/history.js +19 -13
- package/dist-lib/lib/assistant/message.d.ts +6 -1
- package/dist-lib/lib/assistant/message.js +22 -2
- package/dist-lib/lib/assistant/model-round.d.ts +5 -0
- package/dist-lib/lib/assistant/model-round.js +46 -18
- package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
- package/dist-lib/lib/assistant/pdf-budget.js +10 -11
- package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant/round-host.js +5 -3
- package/dist-lib/lib/assistant-client.d.ts +2 -1
- package/dist-lib/lib/assistant-client.js +24 -6
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +8 -5
- package/dist-lib/lib/assistant.d.ts +20 -7
- package/dist-lib/lib/assistant.js +468 -240
- package/dist-lib/lib/client.js +10 -1
- package/dist-lib/lib/compaction.d.ts +3 -1
- package/dist-lib/lib/compaction.js +18 -4
- package/dist-lib/lib/constants.d.ts +3 -0
- package/dist-lib/lib/constants.js +3 -0
- package/dist-lib/lib/core/chat-activity.js +13 -16
- package/dist-lib/lib/core/chat-copy.js +1 -1
- package/dist-lib/lib/core/chat-fork.js +2 -1
- package/dist-lib/lib/core/chat-move.d.ts +4 -0
- package/dist-lib/lib/core/chat-move.js +42 -11
- package/dist-lib/lib/core/chat.d.ts +0 -1
- package/dist-lib/lib/core/chat.js +1 -1
- package/dist-lib/lib/core/chunk.d.ts +14 -3
- package/dist-lib/lib/core/chunk.js +112 -22
- package/dist-lib/lib/core/collections.d.ts +12 -1
- package/dist-lib/lib/core/collections.js +12 -1
- package/dist-lib/lib/core/config-value.d.ts +6 -0
- package/dist-lib/lib/core/config-value.js +14 -0
- package/dist-lib/lib/core/contract/activity.d.ts +3 -2
- package/dist-lib/lib/core/contract/activity.js +39 -21
- package/dist-lib/lib/core/contract/agent-join.js +15 -19
- package/dist-lib/lib/core/contract/agents.js +8 -7
- package/dist-lib/lib/core/contract/chat.d.ts +3 -0
- package/dist-lib/lib/core/contract/chat.js +12 -1
- package/dist-lib/lib/core/contract/chunks.js +16 -6
- package/dist-lib/lib/core/contract/devices.js +6 -4
- package/dist-lib/lib/core/contract/index.js +2 -1
- package/dist-lib/lib/core/contract/invites.js +12 -31
- package/dist-lib/lib/core/contract/joined-meshes.js +2 -1
- package/dist-lib/lib/core/contract/mesh-rules.js +4 -0
- package/dist-lib/lib/core/contract/model-requests.js +7 -6
- package/dist-lib/lib/core/contract/project-device-keys.js +15 -6
- package/dist-lib/lib/core/contract/project-host-keys.js +5 -4
- package/dist-lib/lib/core/contract/projects.js +12 -13
- package/dist-lib/lib/core/contract/public-seeds.js +5 -5
- package/dist-lib/lib/core/contract/roles.js +2 -1
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -1
- package/dist-lib/lib/core/contract/run-requests.js +19 -13
- package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
- package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
- package/dist-lib/lib/core/contract/workspace.js +4 -10
- package/dist-lib/lib/core/contract/writers.js +20 -6
- package/dist-lib/lib/core/devices.d.ts +2 -1
- package/dist-lib/lib/core/devices.js +4 -2
- package/dist-lib/lib/core/document-attachment.d.ts +6 -3
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/attachments.js +3 -3
- package/dist-lib/lib/core/files/index.d.ts +8 -5
- package/dist-lib/lib/core/files/index.js +109 -18
- package/dist-lib/lib/core/files/processing/index.d.ts +1 -1
- package/dist-lib/lib/core/files/processing/index.js +8 -2
- package/dist-lib/lib/core/files/processing/source.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/source.js +3 -0
- package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/state.js +7 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/build.js +14 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.js +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +7 -1
- package/dist-lib/lib/core/first-event.d.ts +6 -0
- package/dist-lib/lib/core/first-event.js +13 -0
- package/dist-lib/lib/core/invite.d.ts +2 -2
- package/dist-lib/lib/core/invite.js +5 -4
- package/dist-lib/lib/core/local/index.d.ts +10 -1
- package/dist-lib/lib/core/local/index.js +105 -10
- package/dist-lib/lib/core/mesh.d.ts +15 -4
- package/dist-lib/lib/core/mesh.js +128 -57
- package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
- package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
- package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
- package/dist-lib/lib/core/migrations/file-index.js +6 -2
- package/dist-lib/lib/core/migrations/index.d.ts +2 -1
- package/dist-lib/lib/core/migrations/index.js +66 -9
- package/dist-lib/lib/core/notification.js +96 -23
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
- package/dist-lib/lib/core/pdf/rasterize.js +72 -22
- package/dist-lib/lib/core/peer-duplex.js +57 -5
- package/dist-lib/lib/core/public-mesh.js +8 -2
- package/dist-lib/lib/core/run-requests.js +2 -3
- package/dist-lib/lib/core/senders.d.ts +7 -6
- package/dist-lib/lib/core/senders.js +23 -30
- package/dist-lib/lib/core/shared-agents.d.ts +1 -0
- package/dist-lib/lib/core/shared-agents.js +47 -19
- package/dist-lib/lib/core/sync/background.d.ts +3 -15
- package/dist-lib/lib/core/sync/background.js +25 -102
- package/dist-lib/lib/core/sync/devices.js +6 -3
- package/dist-lib/lib/core/sync/index.js +12 -2
- package/dist-lib/lib/core/sync-status.js +3 -3
- package/dist-lib/lib/core.d.ts +59 -10
- package/dist-lib/lib/core.js +706 -174
- package/dist-lib/lib/credential-pusher.d.ts +11 -7
- package/dist-lib/lib/credential-pusher.js +40 -8
- package/dist-lib/lib/device-tree.d.ts +18 -0
- package/dist-lib/lib/device-tree.js +32 -0
- package/dist-lib/lib/doctor/redact.js +1 -1
- package/dist-lib/lib/errors.d.ts +4 -0
- package/dist-lib/lib/errors.js +4 -0
- package/dist-lib/lib/fs-drive-bridge.d.ts +7 -0
- package/dist-lib/lib/fs-drive-bridge.js +308 -30
- package/dist-lib/lib/integrations/setup-routes.js +7 -5
- package/dist-lib/lib/knowledge-search.d.ts +1 -1
- package/dist-lib/lib/knowledge-search.js +5 -46
- package/dist-lib/lib/kv-cache-retention.d.ts +35 -0
- package/dist-lib/lib/kv-cache-retention.js +98 -0
- package/dist-lib/lib/latest-frame.d.ts +10 -0
- package/dist-lib/lib/latest-frame.js +48 -0
- package/dist-lib/lib/model-download.d.ts +6 -1
- package/dist-lib/lib/model-download.js +32 -17
- package/dist-lib/lib/oauth/resolve.js +13 -5
- package/dist-lib/lib/reconnecting-watch.js +19 -11
- package/dist-lib/lib/run-notices.d.ts +2 -0
- package/dist-lib/lib/run-notices.js +5 -0
- package/dist-lib/lib/run-replay.d.ts +13 -1
- package/dist-lib/lib/run-replay.js +58 -21
- package/dist-lib/schema/build.js +66 -57
- package/dist-lib/schema/content.js +2 -1
- package/dist-lib/schema/engine.js +37 -5
- package/dist-lib/schema/field-order.js +7 -1
- package/dist-lib/schema/indexes.d.ts +4 -0
- package/dist-lib/schema/indexes.js +18 -0
- package/dist-lib/schema/local.js +9 -2
- package/dist-lib/schema/qvac.js +112 -4
- package/dist-lib/spec/bind.js +3 -0
- package/dist-lib/spec/calls.js +1 -0
- package/dist-lib/spec/capabilities.d.ts +2 -0
- package/dist-lib/spec/hrpc/hrpc.json +13 -0
- package/dist-lib/spec/hrpc/index.js +15 -3
- package/dist-lib/spec/hrpc/messages.js +1475 -1392
- package/dist-lib/spec/hyperschema/index.js +1475 -1392
- package/dist-lib/spec/hyperschema/schema.json +86 -1
- package/dist-lib/spec/hyperschema/types.d.ts +21 -1
- package/dist-lib/spec/local/hyperdb/messages.js +110 -54
- package/dist-lib/spec/local/hyperschema/index.js +56 -28
- package/dist-lib/spec/local/hyperschema/schema.json +55 -20
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -1
- package/dist-lib/spec/mesh/hyperdb/db.json +255 -51
- package/dist-lib/spec/mesh/hyperdb/index.js +1063 -404
- package/dist-lib/spec/mesh/hyperdb/messages.js +1647 -1214
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +62 -56
- package/dist-lib/spec/mesh/hyperdispatch/index.js +156 -139
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +787 -629
- package/dist-lib/spec/mesh/hyperschema/index.js +787 -629
- package/dist-lib/spec/mesh/hyperschema/schema.json +283 -174
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +29 -1
- package/package.json +23 -17
- package/worker/assistant-sidecar.mjs +3 -0
- package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
- package/dist-lib/lib/core/files/processing/settle.js +0 -86
- package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
- package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
|
@@ -9,14 +9,14 @@ import HypercoreId from 'hypercore-id-encoding';
|
|
|
9
9
|
import ReadyResource from 'ready-resource';
|
|
10
10
|
import { Files } from './files/index.js';
|
|
11
11
|
import MeshDefinition from '../../spec/mesh/hyperdb/index.js';
|
|
12
|
-
import { Router, encode } from '../../spec/mesh/hyperdispatch/index.js';
|
|
12
|
+
import { Router, encode, version } from '../../spec/mesh/hyperdispatch/index.js';
|
|
13
13
|
import { syncChanges } from './sync/index.js';
|
|
14
14
|
import { ViewUpdates } from './sync/view-updates.js';
|
|
15
15
|
import { BackgroundSync } from './sync/background.js';
|
|
16
16
|
import { buildDefaultSeed, isOwnKickedRow, isOwnRevokedRow } from './helpers/index.js';
|
|
17
17
|
import { raceResource } from '../race-resource.js';
|
|
18
18
|
import { signDeviceAttestation, signWriterClaim, verifyWriterClaim } from './device-attestation.js';
|
|
19
|
-
import { devicesByWriterKey, liveDeviceForWriter } from './contract/activity.js';
|
|
19
|
+
import { devicesByWriterKey, devicesOfMember, liveDeviceForWriter } from './contract/activity.js';
|
|
20
20
|
import { MESH_NAME_ROW } from './contract/mesh-name.js';
|
|
21
21
|
import { versionedOp } from './contract/oldest-version.js';
|
|
22
22
|
import { KNOWN_RULES, rulesVersion } from './contract/mesh-rules.js';
|
|
@@ -31,10 +31,9 @@ import { CoreError, isCoreError } from '../errors.js';
|
|
|
31
31
|
import { ACTIVE_AGENTS, ACTIVE_CHATS, DEVICES, LIVE_WRITERS_BY_MOST_RECENT, MEMBERS, MESH_NAMES } from './collections.js';
|
|
32
32
|
const [MESH_KEYPAIR, MESH_ENCRYPTION, MESH_DEVICE] = crypto.namespace('qvac-assistant/mesh', 3);
|
|
33
33
|
const DEFAULT_NAMESPACE = b4a.alloc(32);
|
|
34
|
+
export const SCHEMA_VERSION = version;
|
|
34
35
|
const SLOW_WRITE_MS = 5000;
|
|
35
36
|
const SLOW_DRAIN_MS = 15_000;
|
|
36
|
-
const SYNC_BATCH = 64;
|
|
37
|
-
const SYNC_PAUSE_MS = 100;
|
|
38
37
|
const BLOB_CORE = 'qvac-blobs';
|
|
39
38
|
export function meshPublicKeyFromSeed(seed) {
|
|
40
39
|
return crypto.keyPair(crypto.hash([MESH_KEYPAIR, seed])).publicKey;
|
|
@@ -58,7 +57,7 @@ const WARMUP_COLLECTIONS = [
|
|
|
58
57
|
'@qvac/members',
|
|
59
58
|
'@qvac/active-chats'
|
|
60
59
|
];
|
|
61
|
-
export async function
|
|
60
|
+
export async function isTrustedWriter(snap, writerKey, { ourMemberId }) {
|
|
62
61
|
const device = await liveDeviceForWriter(snap, writerKey);
|
|
63
62
|
if (!device) {
|
|
64
63
|
return false;
|
|
@@ -66,7 +65,7 @@ export async function isOurTrustedWriter(snap, writerKey, ourMemberId) {
|
|
|
66
65
|
if (!ourMemberId) {
|
|
67
66
|
return true;
|
|
68
67
|
}
|
|
69
|
-
return !!device.memberId
|
|
68
|
+
return !!device.memberId;
|
|
70
69
|
}
|
|
71
70
|
export function claimableDevice(row) {
|
|
72
71
|
return {
|
|
@@ -124,6 +123,8 @@ export class Mesh extends ReadyResource {
|
|
|
124
123
|
_slowDrainMs;
|
|
125
124
|
_stallReported = false;
|
|
126
125
|
_admitted = false;
|
|
126
|
+
_refused = false;
|
|
127
|
+
_kickSeen = null;
|
|
127
128
|
_birth = null;
|
|
128
129
|
_rulesRaised = null;
|
|
129
130
|
_autobee = null;
|
|
@@ -288,8 +289,6 @@ export class Mesh extends ReadyResource {
|
|
|
288
289
|
this._scope = ['p2p', 'mesh', b4a.toHex(this._keyPair.publicKey).slice(0, 8)];
|
|
289
290
|
registerRoutes(this.router, { role: this._role });
|
|
290
291
|
this._background = new BackgroundSync(this, {
|
|
291
|
-
batch: opts.syncBatch ?? SYNC_BATCH,
|
|
292
|
-
pauseMs: opts.syncPauseMs ?? SYNC_PAUSE_MS,
|
|
293
292
|
logger: this._log,
|
|
294
293
|
scope: this._scope
|
|
295
294
|
});
|
|
@@ -297,6 +296,7 @@ export class Mesh extends ReadyResource {
|
|
|
297
296
|
async _open() {
|
|
298
297
|
this._openedAt = Date.now();
|
|
299
298
|
this._log?.info(this._scope, 'opening', {
|
|
299
|
+
schema: SCHEMA_VERSION,
|
|
300
300
|
creator: this._key === null,
|
|
301
301
|
role: this._role,
|
|
302
302
|
bootFrom: this._bootFrom?.length ?? null,
|
|
@@ -528,25 +528,33 @@ export class Mesh extends ReadyResource {
|
|
|
528
528
|
}
|
|
529
529
|
async _watchStall(pending, watch) {
|
|
530
530
|
const started = Date.now();
|
|
531
|
-
|
|
532
|
-
const
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
531
|
+
const clock = { reports: 0, timer: null };
|
|
532
|
+
const report = (wait) => {
|
|
533
|
+
clock.timer = setTimeout(() => {
|
|
534
|
+
if (!this.closing) {
|
|
535
|
+
report(wait * 2);
|
|
536
|
+
}
|
|
537
|
+
if (clock.reports === 0 && this._stallReported) {
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
this._stallReported = true;
|
|
541
|
+
clock.reports++;
|
|
542
|
+
this._reportStall(watch, { started, reports: clock.reports }).catch(noop);
|
|
543
|
+
}, wait);
|
|
544
|
+
};
|
|
545
|
+
report(watch.ms);
|
|
540
546
|
try {
|
|
541
547
|
await pending;
|
|
542
|
-
this._endStall(watch, { reported, started, failed: false });
|
|
548
|
+
this._endStall(watch, { reported: clock.reports > 0, started, failed: false });
|
|
543
549
|
}
|
|
544
550
|
catch (err) {
|
|
545
|
-
this._endStall(watch, { reported, started, failed: true, err });
|
|
551
|
+
this._endStall(watch, { reported: clock.reports > 0, started, failed: true, err });
|
|
546
552
|
throw err;
|
|
547
553
|
}
|
|
548
554
|
finally {
|
|
549
|
-
|
|
555
|
+
if (clock.timer) {
|
|
556
|
+
clearTimeout(clock.timer);
|
|
557
|
+
}
|
|
550
558
|
}
|
|
551
559
|
}
|
|
552
560
|
_endStall({ what, op }, { reported, started, failed, err }) {
|
|
@@ -562,23 +570,39 @@ export class Mesh extends ReadyResource {
|
|
|
562
570
|
this._log?.info(this._scope, `${what} settled`, { op, ms });
|
|
563
571
|
}
|
|
564
572
|
}
|
|
565
|
-
async _reportStall({ what, op }, started) {
|
|
566
|
-
const { stats, system } = this.autobee;
|
|
573
|
+
async _reportStall({ what, op }, { started, reports }) {
|
|
574
|
+
const { stats, system, writable } = this.autobee;
|
|
567
575
|
const missing = await this._coresNotHeld([system.bee, this.autobee.bee]).catch(noop);
|
|
576
|
+
const fastForwarding = await this._fastForwardHeld().catch(noop);
|
|
577
|
+
const writers = writable ? undefined : await this._writersHeld().catch(noop);
|
|
568
578
|
this._log?.warn(this._scope, `${what} still pending`, {
|
|
569
579
|
op,
|
|
580
|
+
reports,
|
|
570
581
|
afterMs: Date.now() - started,
|
|
571
|
-
writable
|
|
582
|
+
writable,
|
|
572
583
|
interrupted: !!this.autobee.interrupted,
|
|
573
584
|
drains: stats.drains,
|
|
574
585
|
applies: stats.applies,
|
|
575
586
|
appends: stats.appends,
|
|
576
587
|
fastForwards: stats.fastForwards,
|
|
588
|
+
fastForwarding,
|
|
577
589
|
localLength: this.autobee.local.length,
|
|
578
590
|
missing,
|
|
591
|
+
writers,
|
|
579
592
|
blindPeers: this.blindPeerStats()
|
|
580
593
|
});
|
|
581
594
|
}
|
|
595
|
+
async _fastForwardHeld() {
|
|
596
|
+
const ff = this.autobee.fastForwarding;
|
|
597
|
+
if (!ff) {
|
|
598
|
+
return null;
|
|
599
|
+
}
|
|
600
|
+
return { to: ff.head.length, ...(await this._coreHeld(ff.head.key)) };
|
|
601
|
+
}
|
|
602
|
+
async _writersHeld() {
|
|
603
|
+
const writers = [...this.autobee.activeWriters];
|
|
604
|
+
return Promise.all(writers.map((writer) => this._coreHeld(writer.core.key)));
|
|
605
|
+
}
|
|
582
606
|
async _coresNotHeld(views) {
|
|
583
607
|
const keys = new Map();
|
|
584
608
|
for (const { context } of views) {
|
|
@@ -593,11 +617,14 @@ export class Mesh extends ReadyResource {
|
|
|
593
617
|
const core = this._store.get({ key, active: false });
|
|
594
618
|
try {
|
|
595
619
|
await core.ready();
|
|
620
|
+
const reach = core.peers.map(remoteReach);
|
|
596
621
|
return {
|
|
597
622
|
key: b4a.toHex(key).slice(0, 8),
|
|
598
623
|
length: core.length,
|
|
599
624
|
held: core.contiguousLength,
|
|
600
|
-
peers: core.peers.length
|
|
625
|
+
peers: core.peers.length,
|
|
626
|
+
peerLength: Math.max(0, ...reach.map((peer) => peer.length)),
|
|
627
|
+
peerHeld: Math.max(0, ...reach.map((peer) => peer.held))
|
|
601
628
|
};
|
|
602
629
|
}
|
|
603
630
|
finally {
|
|
@@ -744,17 +771,15 @@ export class Mesh extends ReadyResource {
|
|
|
744
771
|
throw this.interrupted;
|
|
745
772
|
}
|
|
746
773
|
}
|
|
747
|
-
async waitForWritable(
|
|
774
|
+
async waitForWritable() {
|
|
748
775
|
if (this.closing || this.autobee.writable)
|
|
749
776
|
return;
|
|
750
777
|
await new Promise((resolve) => {
|
|
751
778
|
const done = () => {
|
|
752
|
-
clearTimeout(timer);
|
|
753
779
|
this.autobee.off('writable', done);
|
|
754
780
|
this.off('close', done);
|
|
755
781
|
resolve();
|
|
756
782
|
};
|
|
757
|
-
const timer = timeout === undefined ? undefined : setTimeout(done, timeout);
|
|
758
783
|
this.autobee.on('writable', done);
|
|
759
784
|
this.on('close', done);
|
|
760
785
|
});
|
|
@@ -795,9 +820,9 @@ export class Mesh extends ReadyResource {
|
|
|
795
820
|
return false;
|
|
796
821
|
const snap = reference.db.snapshot();
|
|
797
822
|
try {
|
|
798
|
-
const trusted = await
|
|
823
|
+
const trusted = await isTrustedWriter(snap, key, this);
|
|
799
824
|
if (!trusted) {
|
|
800
|
-
this._log?.debug(this._scope, 'writer is no device of
|
|
825
|
+
this._log?.debug(this._scope, 'writer is no live device of this mesh', {
|
|
801
826
|
writerKey: b4a.toHex(key).slice(0, 16)
|
|
802
827
|
});
|
|
803
828
|
}
|
|
@@ -815,7 +840,7 @@ export class Mesh extends ReadyResource {
|
|
|
815
840
|
for await (const writer of target.db.find(LIVE_WRITERS_BY_MOST_RECENT, {
|
|
816
841
|
reverse: true
|
|
817
842
|
})) {
|
|
818
|
-
if ((await
|
|
843
|
+
if ((await isTrustedWriter(snap, writer.writerKey, this)) ||
|
|
819
844
|
(await this._vouchedByItsRow(target, reference, writer.writerKey))) {
|
|
820
845
|
this._log?.debug(this._scope, 'vouching for a head', {
|
|
821
846
|
writerKey: b4a.toHex(writer.writerKey).slice(0, 16),
|
|
@@ -824,7 +849,7 @@ export class Mesh extends ReadyResource {
|
|
|
824
849
|
return { key: writer.writerKey, length: writer.length };
|
|
825
850
|
}
|
|
826
851
|
}
|
|
827
|
-
this._log?.debug(this._scope, 'vouching for no head: no writer
|
|
852
|
+
this._log?.debug(this._scope, 'vouching for no head: no live writer has been active');
|
|
828
853
|
return null;
|
|
829
854
|
}
|
|
830
855
|
finally {
|
|
@@ -862,8 +887,11 @@ export class Mesh extends ReadyResource {
|
|
|
862
887
|
return { db };
|
|
863
888
|
}
|
|
864
889
|
async _autobeeWarmup(view) {
|
|
890
|
+
let snap = null;
|
|
865
891
|
try {
|
|
866
|
-
const
|
|
892
|
+
const read = view.db.snapshot();
|
|
893
|
+
snap = read;
|
|
894
|
+
const counts = await Promise.all(WARMUP_COLLECTIONS.map((collection) => drain(read.find(collection))));
|
|
867
895
|
const warmed = counts.reduce((total, count) => total + count, 0);
|
|
868
896
|
this._log?.info(this._scope, 'warmed candidate view', {
|
|
869
897
|
warmed,
|
|
@@ -875,6 +903,9 @@ export class Mesh extends ReadyResource {
|
|
|
875
903
|
catch (err) {
|
|
876
904
|
this._log?.warn(this._scope, 'warmup failed', err);
|
|
877
905
|
}
|
|
906
|
+
finally {
|
|
907
|
+
await snap?.close();
|
|
908
|
+
}
|
|
878
909
|
}
|
|
879
910
|
async _autobeeApply(nodes, view, host) {
|
|
880
911
|
const tx = view.db.transaction();
|
|
@@ -983,17 +1014,11 @@ export class Mesh extends ReadyResource {
|
|
|
983
1014
|
}
|
|
984
1015
|
async _readLanding() {
|
|
985
1016
|
const started = Date.now();
|
|
986
|
-
|
|
987
|
-
this.refreshOurMember(),
|
|
988
|
-
this.refreshMembership(),
|
|
989
|
-
this.refreshName()
|
|
990
|
-
]);
|
|
1017
|
+
await Promise.all([this.refreshOurMember(), this.refreshMembership(), this.refreshName()]);
|
|
991
1018
|
if (this.closing) {
|
|
992
1019
|
return;
|
|
993
1020
|
}
|
|
994
|
-
|
|
995
|
-
this.emit('kicked');
|
|
996
|
-
}
|
|
1021
|
+
await this.kickDirty();
|
|
997
1022
|
if (this._membership.unattested > 0) {
|
|
998
1023
|
this.emit('unattested-device');
|
|
999
1024
|
}
|
|
@@ -1005,6 +1030,8 @@ export class Mesh extends ReadyResource {
|
|
|
1005
1030
|
async checkOwnAdmission() {
|
|
1006
1031
|
if (this.autobee.writable) {
|
|
1007
1032
|
this._admitted = true;
|
|
1033
|
+
this._refused = false;
|
|
1034
|
+
await this.kickDirty();
|
|
1008
1035
|
return;
|
|
1009
1036
|
}
|
|
1010
1037
|
if (!this._admitted || !this._deviceId) {
|
|
@@ -1018,33 +1045,68 @@ export class Mesh extends ReadyResource {
|
|
|
1018
1045
|
return;
|
|
1019
1046
|
}
|
|
1020
1047
|
this._admitted = false;
|
|
1021
|
-
this.
|
|
1048
|
+
this._refused = true;
|
|
1049
|
+
await this.kickDirty();
|
|
1050
|
+
}
|
|
1051
|
+
async kickDirty() {
|
|
1052
|
+
const seen = await this._kickState();
|
|
1053
|
+
if (seen === this._kickSeen || this.closing) {
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
this._kickSeen = seen;
|
|
1057
|
+
this.emit('kick-dirty');
|
|
1058
|
+
}
|
|
1059
|
+
async _kickState() {
|
|
1060
|
+
const writable = this.autobee.writable;
|
|
1061
|
+
const own = await this._ownRow();
|
|
1062
|
+
const member = this.ourMemberId;
|
|
1063
|
+
const devices = !writable && member && this._role === 'workspace'
|
|
1064
|
+
? await devicesOfMember(this.view, member)
|
|
1065
|
+
: [];
|
|
1066
|
+
return JSON.stringify([
|
|
1067
|
+
writable,
|
|
1068
|
+
this._refused,
|
|
1069
|
+
member && b4a.toHex(member),
|
|
1070
|
+
own && [
|
|
1071
|
+
own.revoked,
|
|
1072
|
+
own.revokedBy && b4a.toHex(own.revokedBy),
|
|
1073
|
+
own.writerKey && b4a.toHex(own.writerKey)
|
|
1074
|
+
],
|
|
1075
|
+
devices.map((device) => [b4a.toHex(device.id), device.revoked])
|
|
1076
|
+
]);
|
|
1077
|
+
}
|
|
1078
|
+
async readKicked() {
|
|
1079
|
+
if (this.autobee.writable)
|
|
1080
|
+
return false;
|
|
1081
|
+
const own = await this._ownRow();
|
|
1082
|
+
if (own && isOwnKickedRow(this, own)) {
|
|
1083
|
+
return true;
|
|
1084
|
+
}
|
|
1085
|
+
return this._refused;
|
|
1086
|
+
}
|
|
1087
|
+
async checkResumedRefusal() {
|
|
1088
|
+
if (!(await this.ownWriterRefused())) {
|
|
1089
|
+
return false;
|
|
1090
|
+
}
|
|
1091
|
+
this._refused = true;
|
|
1092
|
+
return true;
|
|
1022
1093
|
}
|
|
1023
1094
|
async checkResumedRevocation() {
|
|
1024
|
-
const
|
|
1025
|
-
|
|
1026
|
-
for await (const device of snap.find(DEVICES)) {
|
|
1027
|
-
if (!isOwnRevokedRow(this, device))
|
|
1028
|
-
continue;
|
|
1029
|
-
return isOwnKickedRow(this, device) ? 'kicked' : 'left';
|
|
1030
|
-
}
|
|
1095
|
+
const own = await this._ownRow();
|
|
1096
|
+
if (!own || !isOwnRevokedRow(this, own)) {
|
|
1031
1097
|
return null;
|
|
1032
1098
|
}
|
|
1033
|
-
|
|
1034
|
-
await snap.close();
|
|
1035
|
-
}
|
|
1099
|
+
return isOwnKickedRow(this, own) ? 'kicked' : 'left';
|
|
1036
1100
|
}
|
|
1037
1101
|
async ownWriterRefused() {
|
|
1038
1102
|
const info = await this.autobee.system.get(this.localWriterKey, { unflushed: true });
|
|
1039
1103
|
if (!info?.isRemoved) {
|
|
1040
1104
|
return false;
|
|
1041
1105
|
}
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
}
|
|
1047
|
-
return true;
|
|
1106
|
+
return !(await liveDeviceForWriter(this.view, this.localWriterKey));
|
|
1107
|
+
}
|
|
1108
|
+
async _ownRow() {
|
|
1109
|
+
return this._deviceId ? this.view.get(DEVICES, { id: this._deviceId }) : null;
|
|
1048
1110
|
}
|
|
1049
1111
|
}
|
|
1050
1112
|
export class MeshBatch {
|
|
@@ -1082,6 +1144,15 @@ export class MeshBatch {
|
|
|
1082
1144
|
}
|
|
1083
1145
|
}
|
|
1084
1146
|
function noop() { }
|
|
1147
|
+
function remoteReach(peer) {
|
|
1148
|
+
if (!isRecord(peer)) {
|
|
1149
|
+
return { length: 0, held: 0 };
|
|
1150
|
+
}
|
|
1151
|
+
return {
|
|
1152
|
+
length: typeof peer.remoteLength === 'number' ? peer.remoteLength : 0,
|
|
1153
|
+
held: typeof peer.remoteContiguousLength === 'number' ? peer.remoteContiguousLength : 0
|
|
1154
|
+
};
|
|
1155
|
+
}
|
|
1085
1156
|
function viewCoreOf(head) {
|
|
1086
1157
|
return isRecord(head) && b4a.isBuffer(head.key) ? b4a.toHex(head.key).slice(0, 8) : null;
|
|
1087
1158
|
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { Mesh } from '../mesh.ts';
|
|
2
|
+
export declare const REINDEX_FROM_APP_VERSION = "0.21.0";
|
|
3
|
+
export declare function floorExcludesBefore(range: string | null | undefined, release: string): boolean;
|
|
4
|
+
export declare function backfillChatChunks(mesh: Mesh): Promise<number>;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { Range, Version, constants, satisfies } from 'bare-semver';
|
|
2
|
+
import { CHAT_CHUNKS, CHUNK_COUNTERS, PURGED_CHATS } from '../collections.js';
|
|
3
|
+
import { inRole } from '../contract/roles.js';
|
|
4
|
+
import { releaseVersion } from '../min-version.js';
|
|
5
|
+
export const REINDEX_FROM_APP_VERSION = '0.21.0';
|
|
6
|
+
export function floorExcludesBefore(range, release) {
|
|
7
|
+
let floor;
|
|
8
|
+
try {
|
|
9
|
+
floor = Range.parse(range ?? '');
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
const atLeast = Range.parse(`>=${releaseVersion(release)}`);
|
|
15
|
+
return (floor.comparators.length > 0 &&
|
|
16
|
+
floor.comparators.every((set) => set.some((bound) => LOWER_BOUNDS.has(bound.operator) && satisfies(releaseOf(bound.version), atLeast))));
|
|
17
|
+
}
|
|
18
|
+
const LOWER_BOUNDS = new Set([constants.GTE, constants.GT, constants.EQ]);
|
|
19
|
+
function releaseOf({ major, minor, patch }) {
|
|
20
|
+
return new Version(major, minor, patch);
|
|
21
|
+
}
|
|
22
|
+
export async function backfillChatChunks(mesh) {
|
|
23
|
+
if (!inRole(mesh.role, '@qvac/reindex-chat-chunks')) {
|
|
24
|
+
return 0;
|
|
25
|
+
}
|
|
26
|
+
const counters = await mesh.view.find(CHUNK_COUNTERS).toArray();
|
|
27
|
+
const purged = await mesh.view.find(PURGED_CHATS).toArray();
|
|
28
|
+
let sent = 0;
|
|
29
|
+
for (const chat of [...counters, ...purged]) {
|
|
30
|
+
if (mesh.closing || !mesh.writable)
|
|
31
|
+
break;
|
|
32
|
+
const due = 'count' in chat ? !(await holdsEnds(mesh.view, chat)) : await holdsAny(mesh.view, chat);
|
|
33
|
+
if (due) {
|
|
34
|
+
await mesh.dispatch('reindex-chat-chunks', { chatId: chat.chatId });
|
|
35
|
+
sent++;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return sent;
|
|
39
|
+
}
|
|
40
|
+
async function holdsEnds(db, { chatId, count }) {
|
|
41
|
+
const first = await db.get(CHAT_CHUNKS, { chatId, seq: 0 });
|
|
42
|
+
const last = await db.get(CHAT_CHUNKS, { chatId, seq: count - 1 });
|
|
43
|
+
return first !== null && last !== null;
|
|
44
|
+
}
|
|
45
|
+
async function holdsAny(db, { chatId }) {
|
|
46
|
+
return ((await db.find(CHAT_CHUNKS, { gte: { chatId }, lte: { chatId } }).one()) !== null);
|
|
47
|
+
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
import type { Mesh } from '../mesh.ts';
|
|
2
|
-
export declare function backfillFileIndex(mesh: Mesh): Promise<
|
|
2
|
+
export declare function backfillFileIndex(mesh: Mesh): Promise<number>;
|
|
@@ -3,26 +3,30 @@ const BATCH = 256;
|
|
|
3
3
|
export async function backfillFileIndex(mesh) {
|
|
4
4
|
const index = mesh.fileIndex;
|
|
5
5
|
if (!index) {
|
|
6
|
-
return;
|
|
6
|
+
return 0;
|
|
7
7
|
}
|
|
8
8
|
const snapshot = mesh.view.snapshot();
|
|
9
|
+
let recorded = 0;
|
|
9
10
|
try {
|
|
10
11
|
let batch = [];
|
|
11
12
|
for await (const chunk of attachmentChunks(snapshot)) {
|
|
12
13
|
if (mesh.closing)
|
|
13
|
-
return;
|
|
14
|
+
return recorded;
|
|
14
15
|
batch.push(chunk);
|
|
15
16
|
if (batch.length < BATCH) {
|
|
16
17
|
continue;
|
|
17
18
|
}
|
|
18
19
|
await index.record(batch);
|
|
20
|
+
recorded += batch.length;
|
|
19
21
|
batch = [];
|
|
20
22
|
}
|
|
21
23
|
if (batch.length > 0) {
|
|
22
24
|
await index.record(batch);
|
|
25
|
+
recorded += batch.length;
|
|
23
26
|
}
|
|
24
27
|
}
|
|
25
28
|
finally {
|
|
26
29
|
await snapshot.close();
|
|
27
30
|
}
|
|
31
|
+
return recorded;
|
|
28
32
|
}
|
|
@@ -4,5 +4,6 @@ import { type MembershipMigrationContext } from './membership.ts';
|
|
|
4
4
|
export type MigrationContext = MembershipMigrationContext & AgentCatalogMigrationContext;
|
|
5
5
|
export interface RunMigrationsOptions {
|
|
6
6
|
readonly catalog?: boolean;
|
|
7
|
+
readonly slowMs?: number;
|
|
7
8
|
}
|
|
8
|
-
export declare function runMigrations(mesh: Mesh, context: MigrationContext, { catalog }?: RunMigrationsOptions): Promise<boolean>;
|
|
9
|
+
export declare function runMigrations(mesh: Mesh, context: MigrationContext, { catalog, slowMs }?: RunMigrationsOptions): Promise<boolean>;
|
|
@@ -1,17 +1,74 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
1
2
|
import { inRole } from '../contract/roles.js';
|
|
2
3
|
import { migrateAgentCatalog } from './agent-catalog.js';
|
|
3
4
|
import { migrateAgentToolsScope } from './agent-tools-scope.js';
|
|
4
5
|
import { migrateMembership } from './membership.js';
|
|
5
6
|
import { sealDeviceRows } from './seal.js';
|
|
6
7
|
import { reindexAgents } from './agent-indexes.js';
|
|
7
|
-
|
|
8
|
-
export async function runMigrations(mesh, context, { catalog = true } = {}) {
|
|
9
|
-
const devices = await reindexDevices(mesh);
|
|
10
|
-
const membership = await migrateMembership(mesh, context);
|
|
8
|
+
const SLOW_MIGRATIONS_MS = 15_000;
|
|
9
|
+
export async function runMigrations(mesh, context, { catalog = true, slowMs = SLOW_MIGRATIONS_MS } = {}) {
|
|
11
10
|
const holdsAgents = mesh.role !== 'public' && inRole(mesh.role, '@qvac/put-agent');
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
11
|
+
const steps = [
|
|
12
|
+
{ name: 'membership', migrate: () => migrateMembership(mesh, context) },
|
|
13
|
+
...(holdsAgents && catalog
|
|
14
|
+
? [{ name: 'agent-catalog', migrate: () => migrateAgentCatalog(mesh, context) }]
|
|
15
|
+
: []),
|
|
16
|
+
{ name: 'seal', migrate: () => sealDeviceRows(mesh, context) },
|
|
17
|
+
{ name: 'reindex-agents', migrate: () => reindexAgents(mesh) },
|
|
18
|
+
...(holdsAgents
|
|
19
|
+
? [{ name: 'agent-tools-scope', migrate: () => migrateAgentToolsScope(mesh) }]
|
|
20
|
+
: [])
|
|
21
|
+
];
|
|
22
|
+
return (await runLogged(mesh, steps, { log: context.log, wait: slowMs })).length > 0;
|
|
23
|
+
}
|
|
24
|
+
async function runLogged(mesh, steps, report) {
|
|
25
|
+
const run = {
|
|
26
|
+
mesh: b4a.toHex(mesh.key).slice(0, 8),
|
|
27
|
+
started: Date.now(),
|
|
28
|
+
step: null,
|
|
29
|
+
timer: null,
|
|
30
|
+
wrote: [],
|
|
31
|
+
ms: {}
|
|
32
|
+
};
|
|
33
|
+
reportWhileSlow(mesh, run, report);
|
|
34
|
+
try {
|
|
35
|
+
await runSteps(steps, run);
|
|
36
|
+
}
|
|
37
|
+
finally {
|
|
38
|
+
if (run.timer) {
|
|
39
|
+
clearTimeout(run.timer);
|
|
40
|
+
}
|
|
41
|
+
report.log?.info(['p2p', 'mesh'], 'migrations ran', {
|
|
42
|
+
mesh: run.mesh,
|
|
43
|
+
ms: Date.now() - run.started,
|
|
44
|
+
wrote: run.wrote,
|
|
45
|
+
steps: run.ms,
|
|
46
|
+
failed: run.step
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return run.wrote;
|
|
50
|
+
}
|
|
51
|
+
async function runSteps(steps, run) {
|
|
52
|
+
for (const { name, migrate } of steps) {
|
|
53
|
+
run.step = name;
|
|
54
|
+
const at = Date.now();
|
|
55
|
+
if (await migrate()) {
|
|
56
|
+
run.wrote.push(name);
|
|
57
|
+
}
|
|
58
|
+
run.ms[name] = Date.now() - at;
|
|
59
|
+
}
|
|
60
|
+
run.step = null;
|
|
61
|
+
}
|
|
62
|
+
function reportWhileSlow(mesh, run, { log, wait }) {
|
|
63
|
+
run.timer = setTimeout(() => {
|
|
64
|
+
if (mesh.closing) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
log?.warn(['p2p', 'mesh'], 'migrations still running', {
|
|
68
|
+
mesh: run.mesh,
|
|
69
|
+
step: run.step,
|
|
70
|
+
afterMs: Date.now() - run.started
|
|
71
|
+
});
|
|
72
|
+
reportWhileSlow(mesh, run, { log, wait: wait * 2 });
|
|
73
|
+
}, wait);
|
|
17
74
|
}
|