@qvac/core 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +19 -2
  2. package/dist-lib/lib/assistant.d.ts +1 -0
  3. package/dist-lib/lib/assistant.js +22 -12
  4. package/dist-lib/lib/core/chat.js +9 -4
  5. package/dist-lib/lib/core/collections.d.ts +1 -0
  6. package/dist-lib/lib/core/collections.js +1 -0
  7. package/dist-lib/lib/core/contract/chat.d.ts +2 -0
  8. package/dist-lib/lib/core/contract/chat.js +67 -1
  9. package/dist-lib/lib/core/contract/chunks.js +5 -0
  10. package/dist-lib/lib/core/contract/compute.d.ts +1 -1
  11. package/dist-lib/lib/core/contract/compute.js +2 -1
  12. package/dist-lib/lib/core/contract/tool-activity.js +5 -0
  13. package/dist-lib/lib/core/files/index.d.ts +2 -0
  14. package/dist-lib/lib/core/files/index.js +7 -1
  15. package/dist-lib/lib/core/files/local-index.d.ts +4 -1
  16. package/dist-lib/lib/core/files/processing/index.d.ts +3 -0
  17. package/dist-lib/lib/core/files/processing/index.js +9 -0
  18. package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
  19. package/dist-lib/lib/core/files/processing/state.js +5 -0
  20. package/dist-lib/lib/core/files/rag/index.d.ts +2 -0
  21. package/dist-lib/lib/core/files/rag/index.js +6 -0
  22. package/dist-lib/lib/core/files/records/index.d.ts +1 -0
  23. package/dist-lib/lib/core/files/records/index.js +1 -0
  24. package/dist-lib/lib/core/files/records/purge.d.ts +4 -0
  25. package/dist-lib/lib/core/files/records/purge.js +108 -0
  26. package/dist-lib/lib/core/files/records/reclaim.js +5 -3
  27. package/dist-lib/lib/core/files/store/blob.d.ts +1 -0
  28. package/dist-lib/lib/core/files/store/blob.js +29 -0
  29. package/dist-lib/lib/core/files/store/index.d.ts +1 -0
  30. package/dist-lib/lib/core/files/store/index.js +6 -1
  31. package/dist-lib/lib/core/local/files.d.ts +2 -0
  32. package/dist-lib/lib/core/local/files.js +31 -0
  33. package/dist-lib/lib/core/sync/index.js +14 -1
  34. package/dist-lib/lib/core.js +22 -2
  35. package/dist-lib/lib/credential-pusher.d.ts +8 -3
  36. package/dist-lib/lib/credential-pusher.js +4 -1
  37. package/dist-lib/schema/qvac.js +10 -0
  38. package/dist-lib/spec/mesh/hyperdb/db.json +16 -1
  39. package/dist-lib/spec/mesh/hyperdb/index.js +71 -2
  40. package/dist-lib/spec/mesh/hyperdb/messages.js +400 -221
  41. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
  42. package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
  43. package/dist-lib/spec/mesh/hyperdispatch/messages.js +117 -2
  44. package/dist-lib/spec/mesh/hyperschema/index.js +117 -2
  45. package/dist-lib/spec/mesh/hyperschema/schema.json +65 -1
  46. package/dist-lib/spec/mesh/hyperschema/types.d.ts +19 -1
  47. package/package.json +3 -3
  48. package/tui/boot.mjs +1 -3
@@ -105,3 +105,32 @@ async function clearBlobRef(mesh, reference) {
105
105
  await remote.close();
106
106
  }
107
107
  }
108
+ function compactable(core) {
109
+ return 'compact' in core && typeof core.compact === 'function';
110
+ }
111
+ async function compact(core) {
112
+ if (!compactable(core)) {
113
+ throw new Error('this hypercore cannot compact');
114
+ }
115
+ await core.compact();
116
+ }
117
+ export async function compactBlobCores(mesh, coreKeys) {
118
+ for (const key of coreKeys) {
119
+ try {
120
+ if (b4a.equals(key, mesh.blobs.core.key)) {
121
+ await compact(mesh.blobs.core);
122
+ continue;
123
+ }
124
+ const remote = await mesh.openBlobs(key);
125
+ try {
126
+ await compact(remote.core);
127
+ }
128
+ finally {
129
+ await remote.close();
130
+ }
131
+ }
132
+ catch (err) {
133
+ logger.warn('sync', 'blob compaction failed', { coreKey: b4a.toHex(key), err });
134
+ }
135
+ }
136
+ }
@@ -26,6 +26,7 @@ export declare class BlobStore extends EventEmitter {
26
26
  readBytes(reference: string, options?: Parameters<typeof readBlobReferenceBytes>[2]): Promise<Buffer<ArrayBufferLike>>;
27
27
  download(reference: string, options: Parameters<typeof downloadBlobReference>[2]): Promise<void>;
28
28
  clear(reference: string, id: string): Promise<boolean>;
29
+ compact(coreKeys: ReadonlyArray<Buffer>): Promise<void>;
29
30
  available(reference: string): Promise<boolean>;
30
31
  sweep(references: string[], coreLength: number): Promise<void>;
31
32
  close(): Promise<void>;
@@ -2,7 +2,7 @@ import b4a from 'b4a';
2
2
  import EventEmitter from 'bare-events';
3
3
  import { sweepUnreferenced } from './sweep.js';
4
4
  import { inlineReference } from './inline.js';
5
- import { clearBlob, stringifyBlobId, writeBlob } from './blob.js';
5
+ import { clearBlob, compactBlobCores, stringifyBlobId, writeBlob } from './blob.js';
6
6
  import { parseBlobReference, readBlobReference, readBlobReferenceBytes, downloadBlobReference } from './reference.js';
7
7
  export { CoreBlobServer } from './server.js';
8
8
  export { compareBlobReferences, isInvalidBlobReferenceError, parseBlobReference, parseBlobReferenceCoreKey } from './reference.js';
@@ -44,6 +44,11 @@ export class BlobStore extends EventEmitter {
44
44
  clear(reference, id) {
45
45
  return clearBlob(this, reference, id);
46
46
  }
47
+ async compact(coreKeys) {
48
+ if (this._closed)
49
+ return;
50
+ await compactBlobCores(this, coreKeys);
51
+ }
47
52
  async available(reference) {
48
53
  if (this._closed)
49
54
  return false;
@@ -14,6 +14,8 @@ export declare class LocalFiles {
14
14
  touch(key: FileKey, now?: number): Promise<void>;
15
15
  private _write;
16
16
  markCleared(file: FileKey): Promise<void>;
17
+ retireChats(meshKey: Buffer, purged: ReadonlyMap<string, number>): Promise<void>;
18
+ forget(file: FileKey): Promise<void>;
17
19
  meshOf(coreKey: Buffer, blockOffset: number): Promise<Buffer | null>;
18
20
  backfilled(meshKey: Buffer): Promise<boolean>;
19
21
  markBackfilled(meshKey: Buffer): Promise<void>;
@@ -83,6 +83,37 @@ export class LocalFiles {
83
83
  await tx.flush();
84
84
  });
85
85
  }
86
+ retireChats(meshKey, purged) {
87
+ return this._locked(async () => {
88
+ const tx = this._db.transaction();
89
+ let changed = false;
90
+ for (const row of await tx.find(LOCAL_FILES, meshRange(meshKey)).toArray()) {
91
+ const at = row.chatId ? purged.get(row.chatId) : undefined;
92
+ if (!at || row.deletedAt) {
93
+ continue;
94
+ }
95
+ await tx.insert(LOCAL_FILES, { ...row, deletedAt: at });
96
+ changed = true;
97
+ }
98
+ if (changed) {
99
+ await tx.flush();
100
+ }
101
+ else {
102
+ await tx.close();
103
+ }
104
+ });
105
+ }
106
+ forget(file) {
107
+ return this._locked(async () => {
108
+ const tx = this._db.transaction();
109
+ if (!(await tx.get(LOCAL_FILES, keyOf(file)))) {
110
+ await tx.close();
111
+ return;
112
+ }
113
+ await tx.delete(LOCAL_FILES, keyOf(file));
114
+ await tx.flush();
115
+ });
116
+ }
86
117
  async meshOf(coreKey, blockOffset) {
87
118
  const key = { coreKey, blockOffset };
88
119
  const row = await this._db.findOne(LOCAL_FILES_BY_BLOB, { gte: key, lte: key });
@@ -2,7 +2,7 @@ import { FileChanges } from './files.js';
2
2
  import { syncDeviceRevocation, syncOwnMember } from './devices.js';
3
3
  import { syncChatTombstone } from './chats.js';
4
4
  import { syncNotificationChunk } from './chunks.js';
5
- import { AGENTS, CHATS, CHUNKS, CHUNK_COUNTERS, DEVICES, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, MEMBERS } from '../collections.js';
5
+ import { AGENTS, CHATS, CHUNKS, CHUNK_COUNTERS, DEVICES, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, MEMBERS, PURGED_CHATS } from '../collections.js';
6
6
  export async function syncChanges(mesh, view, from) {
7
7
  if (mesh.closing)
8
8
  return;
@@ -13,6 +13,7 @@ export async function syncChanges(mesh, view, from) {
13
13
  let meshesChanged = false;
14
14
  let grantsChanged = false;
15
15
  const orderDirty = new Set();
16
+ const purged = new Map();
16
17
  let kindChanged = false;
17
18
  const snap = view.db.snapshot();
18
19
  const files = new FileChanges(snap, from);
@@ -21,6 +22,9 @@ export async function syncChanges(mesh, view, from) {
21
22
  if (mesh.closing)
22
23
  return;
23
24
  await files.add(change);
25
+ if (change.type === 'delete') {
26
+ continue;
27
+ }
24
28
  if (changed(change, DEVICES)) {
25
29
  membershipMoved = true;
26
30
  unattestedDevice ||= !change.value.revoked && !change.value.memberId;
@@ -50,6 +54,9 @@ export async function syncChanges(mesh, view, from) {
50
54
  else if (changed(change, KNOWLEDGE_ARTIFACTS)) {
51
55
  await mesh.files.syncArtifact(change.value);
52
56
  }
57
+ else if (changed(change, PURGED_CHATS)) {
58
+ purged.set(change.value.chatId, change.value.at);
59
+ }
53
60
  else if (change.collection === '@qvac/joined-meshes') {
54
61
  meshesChanged = true;
55
62
  }
@@ -65,6 +72,12 @@ export async function syncChanges(mesh, view, from) {
65
72
  finally {
66
73
  await snap.close();
67
74
  }
75
+ if (purged.size) {
76
+ await mesh.fileIndex?.retireChats(purged);
77
+ for (const chatId of purged.keys()) {
78
+ mesh.emit('chat-purged', chatId);
79
+ }
80
+ }
68
81
  if (indexingDirty.size) {
69
82
  mesh.emit('indexing-dirty', indexingDirty);
70
83
  }
@@ -2511,7 +2511,10 @@ export class Core extends ReadyResource {
2511
2511
  if (key) {
2512
2512
  await this.local.files.markCleared(key);
2513
2513
  }
2514
- }
2514
+ },
2515
+ list: () => this.local.files.list(mesh().key),
2516
+ retireChats: (purged) => this.local.files.retireChats(mesh().key, purged),
2517
+ forget: (file) => this.local.files.forget(file)
2515
2518
  };
2516
2519
  }
2517
2520
  async _indexMeshFiles(mesh) {
@@ -2648,6 +2651,23 @@ export class Core extends ReadyResource {
2648
2651
  ?.forget(chatId)
2649
2652
  .catch((error) => this._log?.warn(['p2p', 'mesh'], 'chat-tombstone inspector reclaim failed', error));
2650
2653
  });
2654
+ const reclaimLock = new ScopeLock({ debounce: true });
2655
+ const reclaim = async () => {
2656
+ if (!(await reclaimLock.lock()))
2657
+ return;
2658
+ try {
2659
+ if (!mesh.closing) {
2660
+ await mesh.files.reclaimPurged();
2661
+ }
2662
+ }
2663
+ catch (error) {
2664
+ this._log?.warn(['p2p', 'mesh'], 'purged file reclaim failed', error);
2665
+ }
2666
+ finally {
2667
+ reclaimLock.unlock();
2668
+ }
2669
+ };
2670
+ mesh.on('chat-purged', () => void reclaim());
2651
2671
  const sweepLock = new ScopeLock({ debounce: true });
2652
2672
  const sweep = async () => {
2653
2673
  if (!(await sweepLock.lock()))
@@ -2665,7 +2685,7 @@ export class Core extends ReadyResource {
2665
2685
  }
2666
2686
  };
2667
2687
  void sweep();
2668
- void this._indexMeshFiles(mesh);
2688
+ void this._indexMeshFiles(mesh).then(reclaim);
2669
2689
  mesh.autobee.on('move-to', () => void sweep());
2670
2690
  entry.hostedAgents = new HostedAgents({
2671
2691
  mesh,
@@ -35,10 +35,15 @@ export interface CredentialEngine extends ConfigEngine {
35
35
  }): AsyncIterable<unknown> & {
36
36
  destroy(): void;
37
37
  };
38
- listAgents(req: {}): Promise<{
39
- agents: ReadonlyArray<{
38
+ account(req: {}): Promise<{
39
+ meshId?: string | null;
40
+ }>;
41
+ listAgents(req: {
42
+ meshId: string;
43
+ }): Promise<{
44
+ agents?: ReadonlyArray<{
40
45
  id: string;
41
- }>;
46
+ }> | null;
42
47
  }>;
43
48
  }
44
49
  export interface CredentialTarget {
@@ -254,7 +254,10 @@ export function startCredentialPusher(harness, engine, logger) {
254
254
  const owners = credentialOwners(skills);
255
255
  const specs = oauthSpecs(skills);
256
256
  syncWatches(specs.map((spec) => spec.credentialKey));
257
- const ids = (await engine.listAgents({})).agents.map((agent) => agent.id);
257
+ const { meshId } = await engine.account({});
258
+ const ids = meshId
259
+ ? ((await engine.listAgents({ meshId })).agents ?? []).map((agent) => agent.id)
260
+ : [];
258
261
  const scopes = [undefined, ...ids];
259
262
  const resolution = { engine, last, owners, specs, logger };
260
263
  for (const agentId of scopes) {
@@ -539,6 +539,14 @@ export function registerQvacTypes(ns) {
539
539
  { name: 'id', type: 'string', required: true },
540
540
  { name: 'memberId', type: 'fixed32', required: true }
541
541
  ]);
542
+ versioned('purged-chat', [
543
+ { name: 'chatId', type: 'string', required: true },
544
+ { name: 'at', type: 'uint', required: true }
545
+ ]);
546
+ versioned('purge-chat-operation', [
547
+ { name: 'chatId', type: 'string', required: true },
548
+ { name: 'at', type: 'uint', required: true }
549
+ ]);
542
550
  }
543
551
  export function registerQvacCollections(dbns, helpersPath) {
544
552
  dbns.require(helpersPath);
@@ -867,6 +875,7 @@ export function registerQvacCollections(dbns, helpersPath) {
867
875
  key: ['memberId']
868
876
  });
869
877
  dbns.collections.register({ name: 'mesh-founders', schema: '@qvac/mesh-founder', key: ['id'] });
878
+ dbns.collections.register({ name: 'purged-chats', schema: '@qvac/purged-chat', key: ['chatId'] });
870
879
  }
871
880
  export function registerQvacDispatch(ds) {
872
881
  ds.register({ name: 'append-chunk', requestType: '@qvac/append-chunk-operation' });
@@ -931,4 +940,5 @@ export function registerQvacDispatch(ds) {
931
940
  ds.register({ name: 'vouched-join', requestType: '@qvac/vouched-join' });
932
941
  ds.register({ name: 'reindex-devices', requestType: '@qvac/reindex-devices-operation' });
933
942
  ds.register({ name: 'recover-device', requestType: '@qvac/recovery-claim' });
943
+ ds.register({ name: 'purge-chat', requestType: '@qvac/purge-chat-operation' });
934
944
  }
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": 7,
2
+ "version": 8,
3
3
  "offset": 0,
4
4
  "schema": [
5
5
  {
@@ -892,6 +892,21 @@
892
892
  "id"
893
893
  ],
894
894
  "trigger": null
895
+ },
896
+ {
897
+ "name": "purged-chats",
898
+ "namespace": "qvac",
899
+ "id": 48,
900
+ "type": 1,
901
+ "version": 8,
902
+ "versionField": null,
903
+ "indexes": [],
904
+ "schema": "@qvac/purged-chat",
905
+ "derived": false,
906
+ "key": [
907
+ "chatId"
908
+ ],
909
+ "trigger": null
895
910
  }
896
911
  ]
897
912
  }
@@ -4,7 +4,7 @@
4
4
  import { IndexEncoder, c, b4a } from 'hyperdb/runtime'
5
5
  import { version, getEncoding, setVersion } from './messages.js'
6
6
 
7
- const versions = { schema: version, db: 7 }
7
+ const versions = { schema: version, db: 8 }
8
8
 
9
9
  import * as helpers0 from '../../../schema/indexes.js'
10
10
 
@@ -2871,6 +2871,73 @@ const collection47 = {
2871
2871
  decodedVersion: 0
2872
2872
  }
2873
2873
 
2874
+ // '@qvac/purged-chats' collection key
2875
+ const collection48_key = new IndexEncoder([
2876
+ IndexEncoder.STRING
2877
+ ], { prefix: 48 })
2878
+
2879
+ function collection48_indexify (record) {
2880
+ const a = record.chatId
2881
+ return a === undefined ? [] : [a]
2882
+ }
2883
+
2884
+ // '@qvac/purged-chats' value encoding
2885
+ const collection48_enc = getEncoding('@qvac/purged-chat/hyperdb#48')
2886
+
2887
+ // '@qvac/purged-chats' reconstruction function
2888
+ function collection48_reconstruct (schemaVersion, keyBuf, valueBuf) {
2889
+ const key = collection48_key.decode(keyBuf)
2890
+ setVersion(schemaVersion)
2891
+ const state = { start: 0, end: valueBuf.byteLength, buffer: valueBuf }
2892
+ const type = c.uint.decode(state)
2893
+ if (type !== 0) throw new Error('Unknown collection type: ' + type)
2894
+ collection48.decodedVersion = c.uint.decode(state)
2895
+ const record = collection48_enc.decode(state)
2896
+ record.chatId = key[0]
2897
+ return record
2898
+ }
2899
+ // '@qvac/purged-chats' key reconstruction function
2900
+ function collection48_reconstruct_key (keyBuf) {
2901
+ const key = collection48_key.decode(keyBuf)
2902
+ return {
2903
+ chatId: key[0]
2904
+ }
2905
+ }
2906
+
2907
+ // '@qvac/purged-chats'
2908
+ const collection48 = {
2909
+ name: '@qvac/purged-chats',
2910
+ id: 48,
2911
+ version: 8,
2912
+ encodeKey (record) {
2913
+ const key = [record.chatId]
2914
+ return collection48_key.encode(key)
2915
+ },
2916
+ encodeKeyRange ({ gt, lt, gte, lte } = {}) {
2917
+ return collection48_key.encodeRange({
2918
+ gt: gt ? collection48_indexify(gt) : null,
2919
+ lt: lt ? collection48_indexify(lt) : null,
2920
+ gte: gte ? collection48_indexify(gte) : null,
2921
+ lte: lte ? collection48_indexify(lte) : null
2922
+ })
2923
+ },
2924
+ encodeValue (schemaVersion, collectionVersion, record) {
2925
+ setVersion(schemaVersion)
2926
+ const state = { start: 0, end: 2, buffer: null }
2927
+ collection48_enc.preencode(state, record)
2928
+ state.buffer = b4a.allocUnsafe(state.end)
2929
+ state.buffer[state.start++] = 0
2930
+ state.buffer[state.start++] = collectionVersion
2931
+ collection48_enc.encode(state, record)
2932
+ return state.buffer
2933
+ },
2934
+ trigger: null,
2935
+ reconstruct: collection48_reconstruct,
2936
+ reconstructKey: collection48_reconstruct_key,
2937
+ indexes: [],
2938
+ decodedVersion: 0
2939
+ }
2940
+
2874
2941
  const collections = [
2875
2942
  collection0,
2876
2943
  collection1,
@@ -2896,7 +2963,8 @@ const collections = [
2896
2963
  collection44,
2897
2964
  collection45,
2898
2965
  collection46,
2899
- collection47
2966
+ collection47,
2967
+ collection48
2900
2968
  ]
2901
2969
 
2902
2970
  const indexes = [
@@ -2954,6 +3022,7 @@ function resolveCollection (name) {
2954
3022
  case '@qvac/invite-sequences': return collection45
2955
3023
  case '@qvac/member-removals': return collection46
2956
3024
  case '@qvac/mesh-founders': return collection47
3025
+ case '@qvac/purged-chats': return collection48
2957
3026
  default: return null
2958
3027
  }
2959
3028
  }