@qvac/core 0.11.0 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CHANGELOG.md +148 -1
  2. package/README.md +19 -16
  3. package/dist-lib/index.d.ts +1 -0
  4. package/dist-lib/index.js +1 -0
  5. package/dist-lib/lib/api/chunk-delta.d.ts +6 -2
  6. package/dist-lib/lib/api/chunk-delta.js +69 -7
  7. package/dist-lib/lib/api/chunks.js +10 -4
  8. package/dist-lib/lib/api/config.d.ts +4 -1
  9. package/dist-lib/lib/api/config.js +23 -4
  10. package/dist-lib/lib/api/devices.js +3 -1
  11. package/dist-lib/lib/api/rag.js +11 -7
  12. package/dist-lib/lib/assistant/harness.d.ts +3 -1
  13. package/dist-lib/lib/assistant/harness.js +10 -0
  14. package/dist-lib/lib/assistant/history.d.ts +1 -0
  15. package/dist-lib/lib/assistant/history.js +9 -3
  16. package/dist-lib/lib/assistant/message.js +2 -2
  17. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  18. package/dist-lib/lib/assistant-tree.js +5 -3
  19. package/dist-lib/lib/assistant.d.ts +6 -1
  20. package/dist-lib/lib/assistant.js +77 -24
  21. package/dist-lib/lib/client.js +10 -1
  22. package/dist-lib/lib/compaction.d.ts +2 -1
  23. package/dist-lib/lib/compaction.js +14 -4
  24. package/dist-lib/lib/constants.d.ts +3 -0
  25. package/dist-lib/lib/constants.js +3 -0
  26. package/dist-lib/lib/core/archived-mesh.js +1 -0
  27. package/dist-lib/lib/core/chat-activity.js +12 -15
  28. package/dist-lib/lib/core/chunk.d.ts +14 -3
  29. package/dist-lib/lib/core/chunk.js +112 -22
  30. package/dist-lib/lib/core/collections.d.ts +1 -0
  31. package/dist-lib/lib/core/collections.js +1 -0
  32. package/dist-lib/lib/core/config-value.d.ts +6 -0
  33. package/dist-lib/lib/core/config-value.js +14 -0
  34. package/dist-lib/lib/core/contract/chat.d.ts +1 -0
  35. package/dist-lib/lib/core/contract/chat.js +8 -1
  36. package/dist-lib/lib/core/contract/chunks.js +20 -6
  37. package/dist-lib/lib/core/contract/compute.d.ts +1 -1
  38. package/dist-lib/lib/core/contract/compute.js +2 -1
  39. package/dist-lib/lib/core/contract/devices.js +3 -2
  40. package/dist-lib/lib/core/departures.js +1 -0
  41. package/dist-lib/lib/core/devices.d.ts +2 -1
  42. package/dist-lib/lib/core/devices.js +4 -2
  43. package/dist-lib/lib/core/files/attachments.js +3 -3
  44. package/dist-lib/lib/core/files/index.d.ts +1 -1
  45. package/dist-lib/lib/core/files/index.js +3 -2
  46. package/dist-lib/lib/core/files/processing/index.d.ts +1 -0
  47. package/dist-lib/lib/core/files/processing/index.js +8 -0
  48. package/dist-lib/lib/core/files/processing/settle.d.ts +1 -0
  49. package/dist-lib/lib/core/files/processing/settle.js +20 -13
  50. package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
  51. package/dist-lib/lib/core/files/rag/index.js +1 -1
  52. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -0
  53. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +4 -1
  54. package/dist-lib/lib/core/hosted-agents.d.ts +3 -0
  55. package/dist-lib/lib/core/hosted-agents.js +19 -0
  56. package/dist-lib/lib/core/inference-calls-receiver.js +2 -2
  57. package/dist-lib/lib/core/known-sender.d.ts +3 -0
  58. package/dist-lib/lib/core/known-sender.js +26 -0
  59. package/dist-lib/lib/core/local/index.d.ts +2 -0
  60. package/dist-lib/lib/core/local/index.js +17 -10
  61. package/dist-lib/lib/core/mesh.d.ts +5 -1
  62. package/dist-lib/lib/core/mesh.js +6 -6
  63. package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
  64. package/dist-lib/lib/core/migrations/chat-chunks.js +43 -0
  65. package/dist-lib/lib/core/notification.js +16 -13
  66. package/dist-lib/lib/core/skill-overlay-receiver.js +2 -2
  67. package/dist-lib/lib/core/skill-secrets.js +2 -2
  68. package/dist-lib/lib/core.d.ts +17 -1
  69. package/dist-lib/lib/core.js +55 -5
  70. package/dist-lib/lib/credential-pusher.d.ts +11 -7
  71. package/dist-lib/lib/credential-pusher.js +40 -8
  72. package/dist-lib/lib/device-tree.d.ts +18 -0
  73. package/dist-lib/lib/device-tree.js +32 -0
  74. package/dist-lib/lib/integrations/setup-routes.js +7 -5
  75. package/dist-lib/lib/kv-cache-retention.d.ts +34 -0
  76. package/dist-lib/lib/kv-cache-retention.js +98 -0
  77. package/dist-lib/schema/content.js +2 -1
  78. package/dist-lib/schema/engine.js +20 -3
  79. package/dist-lib/schema/field-order.js +8 -1
  80. package/dist-lib/schema/local.js +7 -3
  81. package/dist-lib/schema/qvac.js +23 -4
  82. package/dist-lib/spec/hrpc/messages.js +1240 -1194
  83. package/dist-lib/spec/hyperschema/index.js +1240 -1194
  84. package/dist-lib/spec/hyperschema/schema.json +49 -1
  85. package/dist-lib/spec/hyperschema/types.d.ts +12 -1
  86. package/dist-lib/spec/local/hyperdb/messages.js +70 -20
  87. package/dist-lib/spec/local/hyperschema/index.js +36 -11
  88. package/dist-lib/spec/local/hyperschema/schema.json +21 -1
  89. package/dist-lib/spec/local/hyperschema/types.d.ts +5 -1
  90. package/dist-lib/spec/mesh/hyperdb/db.json +17 -1
  91. package/dist-lib/spec/mesh/hyperdb/index.js +83 -2
  92. package/dist-lib/spec/mesh/hyperdb/messages.js +602 -264
  93. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
  94. package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
  95. package/dist-lib/spec/mesh/hyperdispatch/messages.js +137 -14
  96. package/dist-lib/spec/mesh/hyperschema/index.js +137 -14
  97. package/dist-lib/spec/mesh/hyperschema/schema.json +79 -1
  98. package/dist-lib/spec/mesh/hyperschema/types.d.ts +21 -1
  99. package/package.json +13 -9
  100. package/tui/app.mjs +743 -266
  101. package/tui/boot.mjs +2 -2
  102. package/tui/lib/connect-sheet.mjs +350 -0
  103. package/tui/lib/setup.mjs +59 -25
  104. package/tui/lib/skill-connect.mjs +145 -0
@@ -1,8 +1,11 @@
1
1
  import b4a from 'b4a';
2
2
  import ScopeLock from 'scope-lock';
3
+ import { DeadlineScheduler } from '../deadline.js';
4
+ import { RetryBudget } from '../retry-budget.js';
3
5
  import { getAgent, listAgents } from './agent.js';
4
6
  import { agentSkillValues } from './skill-secrets.js';
5
7
  const SCOPE = ['p2p', 'skill-values'];
8
+ const RETRY_MS = 1_000;
6
9
  export const LEGACY_HELD_AGENTS_KEY = 'skill-values.held-agents';
7
10
  export function heldAgentsKey(meshKey) {
8
11
  return `${LEGACY_HELD_AGENTS_KEY}.${b4a.toHex(meshKey)}`;
@@ -17,6 +20,8 @@ export class HostedAgents {
17
20
  _log;
18
21
  _heldKey;
19
22
  _lock = new ScopeLock({ debounce: true });
23
+ _retry = new RetryBudget({ retryMs: RETRY_MS });
24
+ _deadline = new DeadlineScheduler(() => this.kick());
20
25
  _closed;
21
26
  _onDirty = () => this.kick();
22
27
  constructor(opts) {
@@ -40,6 +45,7 @@ export class HostedAgents {
40
45
  async close() {
41
46
  this._closed = true;
42
47
  this._mesh.off('agents-dirty', this._onDirty);
48
+ this._deadline.clear();
43
49
  this._lock.destroy();
44
50
  await this._lock.flush();
45
51
  }
@@ -76,13 +82,26 @@ export class HostedAgents {
76
82
  });
77
83
  if (released) {
78
84
  holding.delete(id);
85
+ this._retry.reset(id);
86
+ }
87
+ else {
88
+ this._retry.fail(id, Date.now());
79
89
  }
80
90
  }
81
91
  }
92
+ this._scheduleRetry(new Set(departed.filter((id) => holding.has(id))));
82
93
  if (!sameIds(held, holding)) {
83
94
  await this._local.set(this._heldKey, [...holding]);
84
95
  }
85
96
  }
97
+ _scheduleRetry(pending) {
98
+ this._retry.retain(pending);
99
+ this._deadline.clear();
100
+ const next = this._retry.nextDeadline(pending, Date.now());
101
+ if (next !== null && !this._closed) {
102
+ this._deadline.schedule(next);
103
+ }
104
+ }
86
105
  async _handOver(id, catalog) {
87
106
  const agent = await getAgent(this._mesh, id);
88
107
  if (!agent) {
@@ -1,12 +1,12 @@
1
1
  import b4a from 'b4a';
2
2
  import { getAgent } from './agent.js';
3
3
  import { getChat } from './chat.js';
4
- import { DEVICES } from './collections.js';
5
4
  import { CoreError } from '../errors.js';
5
+ import { knownSender } from './known-sender.js';
6
6
  const MAX_ANSWER_BYTES = 2 * 1024 * 1024;
7
7
  export function createInferenceCallsReceiver(opts) {
8
8
  return async (request, fromDeviceId) => {
9
- const sender = await opts.mesh.view.get(DEVICES, { id: fromDeviceId });
9
+ const sender = await knownSender(opts.mesh, fromDeviceId);
10
10
  if (!sender || sender.revoked) {
11
11
  throw new CoreError('PEER_SENDER_REVOKED');
12
12
  }
@@ -0,0 +1,3 @@
1
+ import type { QvacDevice } from '../../spec/mesh/hyperschema/types.d.ts';
2
+ import type { Mesh } from './mesh.ts';
3
+ export declare function knownSender(mesh: Mesh, deviceId: Uint8Array): Promise<QvacDevice | null>;
@@ -0,0 +1,26 @@
1
+ import { DEVICES } from './collections.js';
2
+ import { ACK_TIMEOUT_MS } from './handoff-channel.js';
3
+ const SENDER_ROW_WAIT_MS = ACK_TIMEOUT_MS / 2;
4
+ export async function knownSender(mesh, deviceId) {
5
+ const read = () => mesh.view.get(DEVICES, { id: deviceId });
6
+ const row = await read();
7
+ if (row) {
8
+ return row;
9
+ }
10
+ return new Promise((resolve) => {
11
+ const settle = (found) => {
12
+ clearTimeout(timer);
13
+ mesh.unwatch(check);
14
+ mesh.off('close', onClose);
15
+ resolve(found);
16
+ };
17
+ const onClose = () => settle(null);
18
+ const check = () => {
19
+ void read().then((found) => found && settle(found), onClose);
20
+ };
21
+ const timer = setTimeout(onClose, SENDER_ROW_WAIT_MS);
22
+ mesh.watch(check);
23
+ mesh.on('close', onClose);
24
+ check();
25
+ });
26
+ }
@@ -1,6 +1,7 @@
1
1
  import CorestoreStorage from 'hypercore-storage';
2
2
  import HyperDB from 'hyperdb';
3
3
  import type { LocalAccountRecovery, LocalAppConfig, LocalCapabilityProfile, LocalChatMove, LocalMeshUpdate, LocalChatNotify, LocalChatOrder, LocalCustomModel, LocalIdentity, LocalMemberProof, LocalMeshDeparture, LocalMeshMembership, LocalModelDownload, LocalModelNotify } from '../../../spec/local/hyperschema/types.d.ts';
4
+ import type { QvacAutobeeHead } from '../../../spec/mesh/hyperschema/types.d.ts';
4
5
  import type { DeviceMetadata } from '../mesh.ts';
5
6
  import { LocalFiles } from './files.ts';
6
7
  export interface ResolvedSession {
@@ -8,6 +9,7 @@ export interface ResolvedSession {
8
9
  readonly baseKey: Buffer | undefined;
9
10
  readonly writerSeed: string | undefined;
10
11
  readonly compute: boolean;
12
+ readonly bootFrom?: QvacAutobeeHead | null;
11
13
  }
12
14
  export interface ArchivedMeshMembership extends LocalMeshMembership {
13
15
  readonly leftAt: number;
@@ -9,6 +9,7 @@ import LocalDatabaseSpec from '../../../spec/local/hyperdb/index.js';
9
9
  import b4a from 'b4a';
10
10
  import { dropMeshFiles, LocalFiles } from './files.js';
11
11
  import { LOCAL_ACCOUNT_RECOVERY, LOCAL_ACTIVE_MESH, LOCAL_APP_CONFIG, LOCAL_CHAT_MOVES, LOCAL_MESH_UPDATES, LOCAL_CHAT_NOTIFY, LOCAL_CHAT_ORDER, LOCAL_CHAT_ORDER_BY_KEY, LOCAL_CONFIG, LOCAL_CUSTOM_MODEL, LOCAL_DEVICE, LOCAL_IDENTITY, LOCAL_MEMBERSHIP, LOCAL_MEMBER_PROOF, LOCAL_MESH_DEPARTURE, LOCAL_MESH_HISTORY, LOCAL_MESH_MEMBERSHIP, LOCAL_MESH_SESSION, LOCAL_MODEL_DOWNLOAD, LOCAL_MODEL_NOTIFY } from '../collections.js';
12
+ import { configValue, readConfigValue } from '../config-value.js';
12
13
  const IDENTITY_ID = 'identity';
13
14
  const MEMBERSHIP_ID = 'membership';
14
15
  const APP_CONFIG_ID = 'default';
@@ -20,7 +21,8 @@ function sessionOf(row) {
20
21
  seed: row.seed,
21
22
  baseKey: row.key ?? undefined,
22
23
  writerSeed: row.writerSeed ?? undefined,
23
- compute: row.compute ?? false
24
+ compute: row.compute ?? false,
25
+ bootFrom: row.bootKey ? { key: row.bootKey, length: row.bootLength ?? 0 } : null
24
26
  };
25
27
  }
26
28
  export function openLocalStore(storage) {
@@ -46,12 +48,12 @@ class LocalStore {
46
48
  }
47
49
  async get(key) {
48
50
  const row = await this._db.get(LOCAL_CONFIG, { key });
49
- return row === null ? null : row.value;
51
+ return row === null ? null : readConfigValue(row);
50
52
  }
51
53
  set(key, value) {
52
54
  return this._locked(async () => {
53
55
  const tx = this._db.transaction();
54
- await tx.insert(LOCAL_CONFIG, { key, value });
56
+ await tx.insert(LOCAL_CONFIG, { key, ...configValue(value) });
55
57
  await tx.flush();
56
58
  });
57
59
  }
@@ -59,7 +61,7 @@ class LocalStore {
59
61
  return this._locked(async () => {
60
62
  const tx = this._db.transaction();
61
63
  for (const { key, value } of entries) {
62
- await tx.insert(LOCAL_CONFIG, { key, value });
64
+ await tx.insert(LOCAL_CONFIG, { key, ...configValue(value) });
63
65
  }
64
66
  await tx.flush();
65
67
  });
@@ -71,10 +73,10 @@ class LocalStore {
71
73
  return null;
72
74
  }
73
75
  const tx = this._db.transaction();
74
- await tx.insert(LOCAL_CONFIG, { key: toKey, value: row.value });
76
+ await tx.insert(LOCAL_CONFIG, { ...row, key: toKey });
75
77
  await tx.delete(LOCAL_CONFIG, { key: fromKey });
76
78
  await tx.flush();
77
- return row.value;
79
+ return readConfigValue(row);
78
80
  });
79
81
  }
80
82
  deleteAll(keys) {
@@ -510,7 +512,9 @@ class LocalStore {
510
512
  seed: session.seed,
511
513
  writerSeed: session.writerSeed,
512
514
  compute: session.compute,
513
- awaitingAdmission: existing?.awaitingAdmission
515
+ awaitingAdmission: existing?.awaitingAdmission,
516
+ bootKey: session.bootFrom?.key,
517
+ bootLength: session.bootFrom?.length
514
518
  });
515
519
  if (minted) {
516
520
  await tx.insert(LOCAL_ACTIVE_MESH, { deviceId, homeMeshKey: key });
@@ -537,7 +541,8 @@ class LocalStore {
537
541
  key: row.key,
538
542
  seed: row.seed,
539
543
  creator: opts.creator,
540
- writerSeed: row.writerSeed
544
+ writerSeed: row.writerSeed,
545
+ compute: row.compute
541
546
  });
542
547
  await tx.flush();
543
548
  });
@@ -590,12 +595,14 @@ class LocalStore {
590
595
  const { awaitingAdmission } = opts;
591
596
  return this._locked(async () => {
592
597
  const existing = await this._db.get(LOCAL_MESH_MEMBERSHIP, { key });
593
- if (!existing || !!existing.awaitingAdmission === awaitingAdmission) {
598
+ const settled = awaitingAdmission || !existing?.bootKey;
599
+ if (!existing || (!!existing.awaitingAdmission === awaitingAdmission && settled)) {
594
600
  return;
595
601
  }
602
+ const { bootKey, bootLength, ...rest } = existing;
596
603
  const tx = this._db.transaction();
597
604
  await tx.insert(LOCAL_MESH_MEMBERSHIP, {
598
- ...existing,
605
+ ...(awaitingAdmission ? existing : rest),
599
606
  awaitingAdmission
600
607
  });
601
608
  await tx.flush();
@@ -46,7 +46,11 @@ export interface MeshOptions {
46
46
  slowDrainMs?: number;
47
47
  }
48
48
  export declare function meshPublicKeyFromSeed(seed: Buffer): Buffer;
49
- export declare function isOurTrustedWriter(snap: HyperDB, writerKey: Buffer, ourMemberId: Buffer | null): Promise<boolean>;
49
+ export interface TrustScope {
50
+ readonly ourMemberId: Buffer | null;
51
+ readonly compute: boolean;
52
+ }
53
+ export declare function isTrustedWriter(snap: HyperDB, writerKey: Buffer, { ourMemberId, compute }: TrustScope): Promise<boolean>;
50
54
  export type BirthRow = {
51
55
  readonly id: Buffer;
52
56
  readonly joinedAt: number;
@@ -36,7 +36,7 @@ const WARMUP_COLLECTIONS = [
36
36
  '@qvac/members',
37
37
  '@qvac/active-chats'
38
38
  ];
39
- export async function isOurTrustedWriter(snap, writerKey, ourMemberId) {
39
+ export async function isTrustedWriter(snap, writerKey, { ourMemberId, compute }) {
40
40
  const device = await liveDeviceForWriter(snap, writerKey);
41
41
  if (!device) {
42
42
  return false;
@@ -44,7 +44,7 @@ export async function isOurTrustedWriter(snap, writerKey, ourMemberId) {
44
44
  if (!ourMemberId) {
45
45
  return true;
46
46
  }
47
- return !!device.memberId && b4a.equals(device.memberId, ourMemberId);
47
+ return !!device.memberId && (!compute || b4a.equals(device.memberId, ourMemberId));
48
48
  }
49
49
  export function claimableDevice(row) {
50
50
  return {
@@ -665,9 +665,9 @@ export class Mesh extends ReadyResource {
665
665
  return false;
666
666
  const snap = reference.db.snapshot();
667
667
  try {
668
- const trusted = await isOurTrustedWriter(snap, key, this._ourMemberId);
668
+ const trusted = await isTrustedWriter(snap, key, this);
669
669
  if (!trusted) {
670
- this._log?.debug(this._scope, 'writer is no device of ours', {
670
+ this._log?.debug(this._scope, 'writer is no live device of this mesh', {
671
671
  writerKey: b4a.toHex(key).slice(0, 16)
672
672
  });
673
673
  }
@@ -685,7 +685,7 @@ export class Mesh extends ReadyResource {
685
685
  for await (const writer of target.db.find(LIVE_WRITERS_BY_MOST_RECENT, {
686
686
  reverse: true
687
687
  })) {
688
- if ((await isOurTrustedWriter(snap, writer.writerKey, this._ourMemberId)) ||
688
+ if ((await isTrustedWriter(snap, writer.writerKey, this)) ||
689
689
  (await this._vouchedByItsRow(target, reference, writer.writerKey))) {
690
690
  this._log?.debug(this._scope, 'vouching for a head', {
691
691
  writerKey: b4a.toHex(writer.writerKey).slice(0, 16),
@@ -694,7 +694,7 @@ export class Mesh extends ReadyResource {
694
694
  return { key: writer.writerKey, length: writer.length };
695
695
  }
696
696
  }
697
- this._log?.debug(this._scope, 'vouching for no head: no writer of ours has been active');
697
+ this._log?.debug(this._scope, 'vouching for no head: no live writer has been active');
698
698
  return null;
699
699
  }
700
700
  finally {
@@ -0,0 +1,4 @@
1
+ import type { Mesh } from '../mesh.ts';
2
+ export declare const REINDEX_FROM_APP_VERSION = "0.21.0";
3
+ export declare function floorExcludesBefore(range: string | null | undefined, release: string): boolean;
4
+ export declare function backfillChatChunks(mesh: Mesh): Promise<number>;
@@ -0,0 +1,43 @@
1
+ import { Range, Version, constants, satisfies } from 'bare-semver';
2
+ import { CHAT_CHUNKS, CHUNK_COUNTERS, PURGED_CHATS } from '../collections.js';
3
+ import { releaseVersion } from '../min-version.js';
4
+ export const REINDEX_FROM_APP_VERSION = '0.21.0';
5
+ export function floorExcludesBefore(range, release) {
6
+ let floor;
7
+ try {
8
+ floor = Range.parse(range ?? '');
9
+ }
10
+ catch {
11
+ return false;
12
+ }
13
+ const atLeast = Range.parse(`>=${releaseVersion(release)}`);
14
+ return (floor.comparators.length > 0 &&
15
+ floor.comparators.every((set) => set.some((bound) => LOWER_BOUNDS.has(bound.operator) && satisfies(releaseOf(bound.version), atLeast))));
16
+ }
17
+ const LOWER_BOUNDS = new Set([constants.GTE, constants.GT, constants.EQ]);
18
+ function releaseOf({ major, minor, patch }) {
19
+ return new Version(major, minor, patch);
20
+ }
21
+ export async function backfillChatChunks(mesh) {
22
+ const counters = await mesh.view.find(CHUNK_COUNTERS).toArray();
23
+ const purged = await mesh.view.find(PURGED_CHATS).toArray();
24
+ let sent = 0;
25
+ for (const chat of [...counters, ...purged]) {
26
+ if (mesh.closing || !mesh.writable)
27
+ break;
28
+ const due = 'count' in chat ? !(await holdsEnds(mesh.view, chat)) : await holdsAny(mesh.view, chat);
29
+ if (due) {
30
+ await mesh.dispatch('reindex-chat-chunks', { chatId: chat.chatId });
31
+ sent++;
32
+ }
33
+ }
34
+ return sent;
35
+ }
36
+ async function holdsEnds(db, { chatId, count }) {
37
+ const first = await db.get(CHAT_CHUNKS, { chatId, seq: 0 });
38
+ const last = await db.get(CHAT_CHUNKS, { chatId, seq: count - 1 });
39
+ return first !== null && last !== null;
40
+ }
41
+ async function holdsAny(db, { chatId }) {
42
+ return ((await db.find(CHAT_CHUNKS, { gte: { chatId }, lte: { chatId } }).one()) !== null);
43
+ }
@@ -1,5 +1,5 @@
1
1
  import { diagramRepairLinks, isDiagramRepairPrompt } from '@qvac/harness/lib/diagram-repair.ts';
2
- import { findChunksOfType } from './chunk.js';
2
+ import { findChunksOfType, findChunksOfTypes } from './chunk.js';
3
3
  import b4a from 'b4a';
4
4
  import { ACTIVE_CHATS, AGENTS, CHUNK_COUNTERS, DEVICES, MEMBERS } from './collections.js';
5
5
  const DEFAULT_LIMIT = 200;
@@ -46,7 +46,7 @@ export async function deriveNotifications(mesh, local, { limit: requestedLimit,
46
46
  }
47
47
  const contentByChat = new Map();
48
48
  for (const [chatId, terminalSeq] of terminalSeqByChat) {
49
- contentByChat.set(chatId, await findChunksOfType(snap, { chatId, type: 'content', lte: terminalSeq }));
49
+ contentByChat.set(chatId, await findChunksOfTypes(snap, { chatId, types: ['content'], lte: terminalSeq }));
50
50
  }
51
51
  const agentNameById = new Map();
52
52
  for (const agentId of new Set(selected.map((candidate) => candidate.agentId))) {
@@ -74,15 +74,18 @@ export async function deriveNotifications(mesh, local, { limit: requestedLimit,
74
74
  async function chatCandidates(snap, chat, { local, deviceId, own }) {
75
75
  const counter = await snap.get(CHUNK_COUNTERS, { chatId: chat.id });
76
76
  const ackedThrough = (await local.getChatNotify(chat.id))?.ackedThrough ?? 0;
77
- const [statuses, approvals, messages] = await Promise.all([
77
+ const [unread, approvals] = await Promise.all([
78
78
  (counter?.count ?? 0) <= ackedThrough
79
79
  ? []
80
- : findChunksOfType(snap, { chatId: chat.id, type: 'run-status', gte: ackedThrough }),
81
- findChunksOfType(snap, { chatId: chat.id, type: 'approval-request' }),
82
- (counter?.count ?? 0) <= ackedThrough
83
- ? []
84
- : findChunksOfType(snap, { chatId: chat.id, type: 'user-message', gte: ackedThrough })
80
+ : findChunksOfTypes(snap, {
81
+ chatId: chat.id,
82
+ types: ['run-status', 'user-message'],
83
+ gte: ackedThrough
84
+ }),
85
+ findChunksOfType(snap, { chatId: chat.id, type: 'approval-request' })
85
86
  ]);
87
+ const statuses = unread.filter((chunk) => chunk.type === 'run-status');
88
+ const messages = unread.filter((chunk) => chunk.type === 'user-message');
86
89
  const terminalStatuses = statuses.filter((chunk) => {
87
90
  const state = chunk.runStatus?.state;
88
91
  return state === 'completed' || state === 'failed';
@@ -148,11 +151,11 @@ async function messageCandidate(snap, chunk, { chat, deviceId, own }) {
148
151
  };
149
152
  }
150
153
  async function diagramRepairRunIds(snap, chatId, toSeq) {
151
- const [users, requests] = await Promise.all([
152
- findChunksOfType(snap, { chatId, type: 'user-message', lte: toSeq }),
153
- findChunksOfType(snap, { chatId, type: 'run-request', lte: toSeq })
154
- ]);
155
- const records = [...users, ...requests];
154
+ const records = await findChunksOfTypes(snap, {
155
+ chatId,
156
+ types: ['user-message', 'run-request'],
157
+ lte: toSeq
158
+ });
156
159
  const repairs = diagramRepairLinks(records, (chunk) => chunk.userMessage?.text ?? null);
157
160
  const runIds = new Set();
158
161
  for (const chunk of records) {
@@ -1,10 +1,10 @@
1
1
  import b4a from 'b4a';
2
2
  import { getAgent } from './agent.js';
3
- import { DEVICES } from './collections.js';
4
3
  import { CoreError } from '../errors.js';
4
+ import { knownSender } from './known-sender.js';
5
5
  export function createSkillOverlayReceiver(opts) {
6
6
  return async (op, fromDeviceId) => {
7
- const sender = await opts.mesh.view.get(DEVICES, { id: fromDeviceId });
7
+ const sender = await knownSender(opts.mesh, fromDeviceId);
8
8
  if (!sender || sender.revoked) {
9
9
  throw new CoreError('PEER_SENDER_REVOKED');
10
10
  }
@@ -4,11 +4,11 @@ import { isDeviceSharedCredentialKey } from '@qvac/harness/lib/tools/skills/scop
4
4
  import { settingConfigKey } from '../credential-pusher.js';
5
5
  import { scopedKey } from '../oauth/keys.js';
6
6
  import { getAgent } from './agent.js';
7
- import { DEVICES } from './collections.js';
8
7
  import { CoreError } from '../errors.js';
8
+ import { knownSender } from './known-sender.js';
9
9
  export function createSkillSecretReceiver(opts) {
10
10
  return async (request, fromDeviceId) => {
11
- const sender = await opts.mesh.view.get(DEVICES, { id: fromDeviceId });
11
+ const sender = await knownSender(opts.mesh, fromDeviceId);
12
12
  if (!sender || sender.revoked) {
13
13
  throw new CoreError('PEER_SENDER_REVOKED');
14
14
  }
@@ -80,6 +80,19 @@ export interface CoreOptions {
80
80
  readonly suspend: () => Promise<void>;
81
81
  readonly resume: () => Promise<void>;
82
82
  };
83
+ network?: NetworkGate;
84
+ }
85
+ export interface NetworkConnection {
86
+ destroy(): void;
87
+ }
88
+ export interface NetworkGate {
89
+ readonly onConnection?: (conn: NetworkConnection) => void;
90
+ readonly onMeshTopic?: (topic: MeshTopic) => void;
91
+ }
92
+ export interface MeshTopic {
93
+ readonly meshId: string;
94
+ readonly topic: Buffer;
95
+ readonly swarm: Hyperswarm;
83
96
  }
84
97
  export declare const DEVICE_UNKNOWN = "This device has no device details: pass `device` when creating Core";
85
98
  export interface SharedMesh {
@@ -131,6 +144,7 @@ export declare class Core extends ReadyResource {
131
144
  private readonly _log;
132
145
  private readonly _spec;
133
146
  private readonly _bootstrapNodes;
147
+ private readonly _network;
134
148
  private readonly _device;
135
149
  private readonly _app;
136
150
  private _minSupportedVersion;
@@ -235,7 +249,7 @@ export declare class Core extends ReadyResource {
235
249
  private _systemInfo;
236
250
  private _statusError;
237
251
  private _upgradeRequired;
238
- constructor(transport: Transport, { storagePath, store, swarm, wakeup, logger, spec, bootstrapNodes, device, app, minSupportedVersion, otaConfigCore, blindPeerKeys, blindPeeringOpts, fileGc, autoDownloadModels, meshSeed, meshKey, additionalNodes, listModels, deleteModel, clearKvCache, deleteChatKvCache, compactChat, modelCatalogWatch, retryModelDownload, transcribe, detectLanguage, previewModelSource, speak, speakStream, embedder, extractors, skills, skillCatalog, tools, hostsModelSource, vectorIndex, apiLifecycle }: CoreOptions);
252
+ constructor(transport: Transport, { storagePath, store, swarm, wakeup, logger, spec, bootstrapNodes, device, app, minSupportedVersion, otaConfigCore, blindPeerKeys, blindPeeringOpts, fileGc, autoDownloadModels, meshSeed, meshKey, additionalNodes, listModels, deleteModel, clearKvCache, deleteChatKvCache, compactChat, modelCatalogWatch, retryModelDownload, transcribe, detectLanguage, previewModelSource, speak, speakStream, embedder, extractors, skills, skillCatalog, tools, hostsModelSource, vectorIndex, apiLifecycle, network }: CoreOptions);
239
253
  get devicePublicKey(): Buffer | null;
240
254
  private _requireDeviceKeyPair;
241
255
  get fsDrives(): DriveHost | null;
@@ -371,6 +385,7 @@ export declare class Core extends ReadyResource {
371
385
  private _pendingUpdatesFor;
372
386
  private _fileIndexFor;
373
387
  private _indexMeshFiles;
388
+ private _backfillChatChunks;
374
389
  private _servedFile;
375
390
  clearOldFiles({ olderThanMs, includeOwn }: ClearOldFilesOptions): Promise<number>;
376
391
  private _startFileGc;
@@ -437,6 +452,7 @@ export declare class Core extends ReadyResource {
437
452
  pushSkillSecret(request: SecretRequest): Promise<void>;
438
453
  private _storeHandedOffSecret;
439
454
  private _followJoinedMeshes;
455
+ private _followJoinedMeshesPass;
440
456
  private _followMeshRows;
441
457
  private _trackFollow;
442
458
  private _retireLeftMesh;
@@ -45,6 +45,7 @@ import { openRecoveryCore, publishRecovery, readRecovery, recoveryKeys } from '.
45
45
  import { claimOwnMembership, legacyNameStamp, namedAt, renameFoundedMember } from './core/membership.js';
46
46
  import { runMigrations } from './core/migrations/index.js';
47
47
  import { backfillFileIndex } from './core/migrations/file-index.js';
48
+ import { REINDEX_FROM_APP_VERSION, backfillChatChunks, floorExcludesBefore } from './core/migrations/chat-chunks.js';
48
49
  import { advertiseDeviceVersion, retractInventory } from './core/devices.js';
49
50
  import { DeviceControl } from './core/device-control.js';
50
51
  import { forwardToDevices, servedToSiblings } from './api/device-routes.js';
@@ -89,6 +90,7 @@ export class Core extends ReadyResource {
89
90
  _log;
90
91
  _spec;
91
92
  _bootstrapNodes;
93
+ _network;
92
94
  _device;
93
95
  _app;
94
96
  _minSupportedVersion;
@@ -193,7 +195,7 @@ export class Core extends ReadyResource {
193
195
  _systemInfo;
194
196
  _statusError;
195
197
  _upgradeRequired;
196
- constructor(transport, { storagePath, store = undefined, swarm = undefined, wakeup = undefined, logger = null, spec = openLocalSpec, bootstrapNodes = undefined, device = undefined, app = undefined, minSupportedVersion = undefined, otaConfigCore = undefined, blindPeerKeys = undefined, blindPeeringOpts = undefined, fileGc = null, autoDownloadModels = undefined, meshSeed = undefined, meshKey = undefined, additionalNodes = undefined, listModels = undefined, deleteModel = undefined, clearKvCache = undefined, deleteChatKvCache = undefined, compactChat = undefined, modelCatalogWatch = undefined, retryModelDownload = undefined, transcribe = undefined, detectLanguage = undefined, previewModelSource = undefined, speak = undefined, speakStream = undefined, embedder = undefined, extractors = undefined, skills = undefined, skillCatalog = undefined, tools = undefined, hostsModelSource = false, vectorIndex = undefined, apiLifecycle = undefined }) {
198
+ constructor(transport, { storagePath, store = undefined, swarm = undefined, wakeup = undefined, logger = null, spec = openLocalSpec, bootstrapNodes = undefined, device = undefined, app = undefined, minSupportedVersion = undefined, otaConfigCore = undefined, blindPeerKeys = undefined, blindPeeringOpts = undefined, fileGc = null, autoDownloadModels = undefined, meshSeed = undefined, meshKey = undefined, additionalNodes = undefined, listModels = undefined, deleteModel = undefined, clearKvCache = undefined, deleteChatKvCache = undefined, compactChat = undefined, modelCatalogWatch = undefined, retryModelDownload = undefined, transcribe = undefined, detectLanguage = undefined, previewModelSource = undefined, speak = undefined, speakStream = undefined, embedder = undefined, extractors = undefined, skills = undefined, skillCatalog = undefined, tools = undefined, hostsModelSource = false, vectorIndex = undefined, apiLifecycle = undefined, network = undefined }) {
197
199
  super();
198
200
  this._transport = transport;
199
201
  this._apiLifecycle = apiLifecycle ?? {
@@ -219,6 +221,7 @@ export class Core extends ReadyResource {
219
221
  this.state = QvacState.RESUME;
220
222
  this._spec = spec;
221
223
  this._bootstrapNodes = bootstrapNodes;
224
+ this._network = network;
222
225
  this._device = device;
223
226
  this._app = app;
224
227
  this._minSupportedVersion = minSupportedVersion;
@@ -774,6 +777,9 @@ export class Core extends ReadyResource {
774
777
  if (withheld) {
775
778
  void this._reopenMesh();
776
779
  }
780
+ for (const { mesh } of this._meshes.values()) {
781
+ void this._backfillChatChunks(mesh);
782
+ }
777
783
  return;
778
784
  }
779
785
  for (const { mesh } of [...this._meshes.values()]) {
@@ -984,6 +990,10 @@ export class Core extends ReadyResource {
984
990
  bootstrap: this._bootstrapNodes,
985
991
  relayThrough: this._bootstrapNodes ? undefined : () => BLIND_RELAY_KEYS
986
992
  });
993
+ const onConnection = this._network?.onConnection;
994
+ if (onConnection) {
995
+ this._swarm.on('connection', (conn) => onConnection(conn));
996
+ }
987
997
  this._swarm.on('connection', this._onSwarmConnection);
988
998
  for (const conn of this._swarm.connections) {
989
999
  if (isPeerConnection(conn)) {
@@ -1050,6 +1060,7 @@ export class Core extends ReadyResource {
1050
1060
  writerSeed: session.writerSeed,
1051
1061
  logger: this._log,
1052
1062
  blindPeerKeys: this._resolveBlindPeerKeys(blindPeerKeys),
1063
+ bootFrom: session.bootFrom,
1053
1064
  trustedDevice: (deviceId) => this._isOurMemberDevice(deviceId),
1054
1065
  wakeup: this._wakeup,
1055
1066
  attachmentUrl: (attachment) => this._linkAttachment(attachment, mesh),
@@ -1491,6 +1502,11 @@ export class Core extends ReadyResource {
1491
1502
  return;
1492
1503
  if (this._swarm) {
1493
1504
  this._swarm.join(mesh.discoveryKey, { server: true, client: true });
1505
+ this._network?.onMeshTopic?.({
1506
+ meshId: this._meshIdOf(mesh),
1507
+ topic: mesh.discoveryKey,
1508
+ swarm: this._swarm
1509
+ });
1494
1510
  this._startBlindPeering(mesh);
1495
1511
  if (this.suspended) {
1496
1512
  void mesh.suspend().catch(noop);
@@ -2075,7 +2091,10 @@ export class Core extends ReadyResource {
2075
2091
  }
2076
2092
  }
2077
2093
  async _joinCandidate(invite, source, deviceId) {
2078
- const session = this.local.createCandidateSession(invite.seed, invite.baseKey, invite.compute);
2094
+ const session = {
2095
+ ...this.local.createCandidateSession(invite.seed, invite.baseKey, invite.compute),
2096
+ bootFrom: invite.fastForward
2097
+ };
2079
2098
  let candidate;
2080
2099
  try {
2081
2100
  candidate = await this._openCandidateMesh(invite, session);
@@ -2283,6 +2302,9 @@ export class Core extends ReadyResource {
2283
2302
  const held = key ? (this._pooled(b4a.toHex(key)) ?? this._opening.get(b4a.toHex(key))) : null;
2284
2303
  const mesh = key ? await this._openPooledMesh(b4a.toHex(key)).catch(noop) : null;
2285
2304
  if (mesh && !mesh.ourMemberId) {
2305
+ if (mesh.writable) {
2306
+ await mesh.autobee.update();
2307
+ }
2286
2308
  await mesh.refreshOurMember();
2287
2309
  }
2288
2310
  if (!mesh?.writable) {
@@ -2304,7 +2326,7 @@ export class Core extends ReadyResource {
2304
2326
  writerSeed: session.writerSeed,
2305
2327
  logger: this._log,
2306
2328
  blindPeerKeys: this._resolveBlindPeerKeys(invite.blindPeers),
2307
- bootFrom: invite.fastForward,
2329
+ bootFrom: session.bootFrom,
2308
2330
  trustedDevice: (deviceId) => this._isOurMemberDevice(deviceId),
2309
2331
  wakeup: this._wakeup,
2310
2332
  attachmentUrl: (attachment) => this._linkAttachment(attachment, candidate),
@@ -2431,11 +2453,11 @@ export class Core extends ReadyResource {
2431
2453
  }
2432
2454
  mesh.on('members-dirty', () => void this._revalidateLoans());
2433
2455
  mesh.on('members-dirty', () => void this._deviceControl.revalidate());
2456
+ mesh.on('members-dirty', () => void this._revokeInMeshes());
2434
2457
  mesh.on('members-dirty', () => {
2435
2458
  if (!this._isAccount(mesh))
2436
2459
  return;
2437
2460
  void this._followAccountName(mesh);
2438
- void this._revokeInMeshes();
2439
2461
  });
2440
2462
  mesh.on('meshes-dirty', (changed) => {
2441
2463
  if (changed || entry.meshesPending) {
@@ -2474,6 +2496,7 @@ export class Core extends ReadyResource {
2474
2496
  this._upgradeRequired = null;
2475
2497
  if (mesh.writable) {
2476
2498
  this._kicked = false;
2499
+ void this.local.markAwaitingAdmission(mesh.key, { awaitingAdmission: false }).catch(noop);
2477
2500
  }
2478
2501
  this._notifyStatus();
2479
2502
  }
@@ -2599,6 +2622,21 @@ export class Core extends ReadyResource {
2599
2622
  }
2600
2623
  }
2601
2624
  }
2625
+ async _backfillChatChunks(mesh) {
2626
+ if (!floorExcludesBefore(this._minSupportedVersion, REINDEX_FROM_APP_VERSION)) {
2627
+ return;
2628
+ }
2629
+ try {
2630
+ await mesh.waitForWritable();
2631
+ await mesh.autobee.update();
2632
+ await backfillChatChunks(mesh);
2633
+ }
2634
+ catch (err) {
2635
+ if (!mesh.closing) {
2636
+ this._log?.warn(['p2p', 'mesh'], 'copying chats into chat-chunks failed', err);
2637
+ }
2638
+ }
2639
+ }
2602
2640
  _servedFile(mesh, coreKey, blockOffset) {
2603
2641
  this.local.files
2604
2642
  .touch({ meshKey: mesh.key, coreKey, blockOffset })
@@ -2767,6 +2805,7 @@ export class Core extends ReadyResource {
2767
2805
  }
2768
2806
  };
2769
2807
  void this._indexMeshFiles(mesh).then(() => Promise.all([reclaim(), sweep()]));
2808
+ void this._backfillChatChunks(mesh);
2770
2809
  mesh.autobee.on('move-to', () => void sweep());
2771
2810
  entry.hostedAgents = new HostedAgents({
2772
2811
  mesh,
@@ -3726,7 +3765,18 @@ export class Core extends ReadyResource {
3726
3765
  });
3727
3766
  return receive(request, fromDeviceId);
3728
3767
  }
3729
- async _followJoinedMeshes(memberMesh) {
3768
+ _followJoinedMeshes(memberMesh) {
3769
+ return this._followJoinedMeshesPass(memberMesh).catch((err) => {
3770
+ const entry = this._meshes.get(this._meshIdOf(memberMesh));
3771
+ if (entry?.mesh === memberMesh) {
3772
+ entry.meshesPending = true;
3773
+ }
3774
+ if (!this.closing) {
3775
+ this._log?.warn(['p2p', 'mesh'], 'following the joined meshes failed', err);
3776
+ }
3777
+ });
3778
+ }
3779
+ async _followJoinedMeshesPass(memberMesh) {
3730
3780
  if (this.closing || !this._isPooled(memberMesh))
3731
3781
  return;
3732
3782
  const entry = this._meshes.get(this._meshIdOf(memberMesh));