@qvac/core 0.3.5 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/CHANGELOG.md +177 -9
  2. package/README.md +19 -17
  3. package/dist-lib/lib/api/agents.d.ts +3 -3
  4. package/dist-lib/lib/api/agents.js +31 -20
  5. package/dist-lib/lib/api/approval-rules.d.ts +2 -2
  6. package/dist-lib/lib/api/approval-rules.js +2 -2
  7. package/dist-lib/lib/api/attachments.d.ts +2 -2
  8. package/dist-lib/lib/api/attachments.js +14 -25
  9. package/dist-lib/lib/api/chat-groups.d.ts +2 -2
  10. package/dist-lib/lib/api/chat-groups.js +9 -9
  11. package/dist-lib/lib/api/chat.d.ts +4 -2
  12. package/dist-lib/lib/api/chat.js +18 -18
  13. package/dist-lib/lib/api/chunks.d.ts +2 -2
  14. package/dist-lib/lib/api/chunks.js +27 -36
  15. package/dist-lib/lib/api/current-mesh.d.ts +7 -15
  16. package/dist-lib/lib/api/current-mesh.js +7 -22
  17. package/dist-lib/lib/api/devices.d.ts +4 -3
  18. package/dist-lib/lib/api/devices.js +18 -18
  19. package/dist-lib/lib/api/execution.d.ts +2 -2
  20. package/dist-lib/lib/api/execution.js +6 -6
  21. package/dist-lib/lib/api/files.d.ts +2 -2
  22. package/dist-lib/lib/api/files.js +4 -4
  23. package/dist-lib/lib/api/index.d.ts +12 -11
  24. package/dist-lib/lib/api/index.js +15 -13
  25. package/dist-lib/lib/api/inference-calls.d.ts +1 -1
  26. package/dist-lib/lib/api/inference-calls.js +2 -2
  27. package/dist-lib/lib/api/knowledge.d.ts +2 -2
  28. package/dist-lib/lib/api/knowledge.js +22 -11
  29. package/dist-lib/lib/api/local-files.d.ts +3 -0
  30. package/dist-lib/lib/api/local-files.js +23 -0
  31. package/dist-lib/lib/api/members.d.ts +6 -2
  32. package/dist-lib/lib/api/members.js +24 -17
  33. package/dist-lib/lib/api/mesh.d.ts +4 -5
  34. package/dist-lib/lib/api/mesh.js +7 -9
  35. package/dist-lib/lib/api/notifications.d.ts +2 -13
  36. package/dist-lib/lib/api/notifications.js +20 -52
  37. package/dist-lib/lib/api/rag.d.ts +2 -2
  38. package/dist-lib/lib/api/rag.js +4 -4
  39. package/dist-lib/lib/api/range.d.ts +11 -0
  40. package/dist-lib/lib/api/range.js +10 -0
  41. package/dist-lib/lib/api/sync.d.ts +2 -2
  42. package/dist-lib/lib/api/sync.js +3 -3
  43. package/dist-lib/lib/api/tool-activity.d.ts +2 -2
  44. package/dist-lib/lib/api/tool-activity.js +11 -10
  45. package/dist-lib/lib/api/tool-configs.d.ts +2 -2
  46. package/dist-lib/lib/api/tool-configs.js +6 -7
  47. package/dist-lib/lib/api/upload-frames.d.ts +9 -0
  48. package/dist-lib/lib/api/upload-frames.js +17 -0
  49. package/dist-lib/lib/api/watchable.d.ts +1 -6
  50. package/dist-lib/lib/api/watchable.js +4 -13
  51. package/dist-lib/lib/assistant/history.d.ts +3 -3
  52. package/dist-lib/lib/assistant-client.js +3 -2
  53. package/dist-lib/lib/assistant-tree.js +1 -1
  54. package/dist-lib/lib/assistant.d.ts +16 -14
  55. package/dist-lib/lib/assistant.js +166 -137
  56. package/dist-lib/lib/collabora/convert.js +1 -1
  57. package/dist-lib/lib/core/agent.d.ts +5 -7
  58. package/dist-lib/lib/core/agent.js +26 -6
  59. package/dist-lib/lib/core/archived-mesh.js +1 -0
  60. package/dist-lib/lib/core/chat-order.d.ts +20 -0
  61. package/dist-lib/lib/core/chat-order.js +96 -0
  62. package/dist-lib/lib/core/chat.d.ts +0 -5
  63. package/dist-lib/lib/core/chat.js +1 -3
  64. package/dist-lib/lib/core/chunk.d.ts +7 -14
  65. package/dist-lib/lib/core/chunk.js +17 -35
  66. package/dist-lib/lib/core/contract/agents.js +8 -0
  67. package/dist-lib/lib/core/contract/compute.d.ts +1 -1
  68. package/dist-lib/lib/core/contract/compute.js +1 -0
  69. package/dist-lib/lib/core/contract/joined-meshes.js +4 -3
  70. package/dist-lib/lib/core/contract/members.js +4 -3
  71. package/dist-lib/lib/core/contract/writers.js +5 -4
  72. package/dist-lib/lib/core/departures.js +5 -4
  73. package/dist-lib/lib/core/device-invites.js +2 -1
  74. package/dist-lib/lib/core/files/attachments.d.ts +8 -1
  75. package/dist-lib/lib/core/files/index.d.ts +4 -4
  76. package/dist-lib/lib/core/files/index.js +18 -8
  77. package/dist-lib/lib/core/files/local-index.d.ts +16 -0
  78. package/dist-lib/lib/core/files/local-index.js +38 -0
  79. package/dist-lib/lib/core/files/processing/document.js +2 -1
  80. package/dist-lib/lib/core/files/processing/ownership.js +6 -3
  81. package/dist-lib/lib/core/files/records/chunks.js +1 -0
  82. package/dist-lib/lib/core/files/records/reclaim.js +1 -0
  83. package/dist-lib/lib/core/files/records/routing.js +2 -1
  84. package/dist-lib/lib/core/files/store/blob.d.ts +1 -1
  85. package/dist-lib/lib/core/files/store/blob.js +6 -3
  86. package/dist-lib/lib/core/files/store/index.d.ts +1 -1
  87. package/dist-lib/lib/core/files/store/reference.js +4 -3
  88. package/dist-lib/lib/core/files/store/server.d.ts +8 -4
  89. package/dist-lib/lib/core/files/store/server.js +16 -12
  90. package/dist-lib/lib/core/files/store/sweep.js +2 -1
  91. package/dist-lib/lib/core/files/upload.d.ts +1 -1
  92. package/dist-lib/lib/core/hosted-agents.js +1 -1
  93. package/dist-lib/lib/core/image/cache-sweep.d.ts +1 -0
  94. package/dist-lib/lib/core/image/cache-sweep.js +7 -3
  95. package/dist-lib/lib/core/local/files.d.ts +24 -0
  96. package/dist-lib/lib/core/local/files.js +154 -0
  97. package/dist-lib/lib/core/local/index.d.ts +21 -5
  98. package/dist-lib/lib/core/local/index.js +75 -45
  99. package/dist-lib/lib/core/membership.d.ts +4 -1
  100. package/dist-lib/lib/core/membership.js +17 -2
  101. package/dist-lib/lib/core/mesh.d.ts +5 -0
  102. package/dist-lib/lib/core/mesh.js +10 -5
  103. package/dist-lib/lib/core/migrations/agent-indexes.d.ts +2 -0
  104. package/dist-lib/lib/core/migrations/agent-indexes.js +30 -0
  105. package/dist-lib/lib/core/migrations/file-index.d.ts +2 -0
  106. package/dist-lib/lib/core/migrations/file-index.js +27 -0
  107. package/dist-lib/lib/core/migrations/index.js +3 -1
  108. package/dist-lib/lib/core/notification.js +5 -5
  109. package/dist-lib/lib/core/range.d.ts +18 -0
  110. package/dist-lib/lib/core/range.js +17 -0
  111. package/dist-lib/lib/core/sync/index.js +39 -32
  112. package/dist-lib/lib/core/sync-status.js +3 -2
  113. package/dist-lib/lib/core/tool-activity.d.ts +2 -5
  114. package/dist-lib/lib/core/tool-activity.js +4 -7
  115. package/dist-lib/lib/core.d.ts +48 -29
  116. package/dist-lib/lib/core.js +465 -359
  117. package/dist-lib/lib/knowledge-search.d.ts +2 -2
  118. package/dist-lib/lib/kv-cache.js +1 -1
  119. package/dist-lib/lib/log.js +2 -3
  120. package/dist-lib/lib/mesh-scoped-engine.d.ts +8 -1
  121. package/dist-lib/lib/mesh-scoped-engine.js +62 -15
  122. package/dist-lib/lib/model-download.d.ts +14 -5
  123. package/dist-lib/lib/model-download.js +25 -17
  124. package/dist-lib/lib/oauth/browser-connect.d.ts +0 -14
  125. package/dist-lib/lib/oauth/browser-connect.js +1 -26
  126. package/dist-lib/lib/oauth/token-exchange.d.ts +1 -2
  127. package/dist-lib/lib/oauth/token-exchange.js +1 -4
  128. package/dist-lib/lib/run-transcript.d.ts +2 -2
  129. package/dist-lib/lib/run-transcript.js +3 -2
  130. package/dist-lib/lib/task.d.ts +1 -0
  131. package/dist-lib/lib/task.js +3 -2
  132. package/dist-lib/schema/build.js +4 -10
  133. package/dist-lib/schema/engine.js +261 -91
  134. package/dist-lib/schema/field-order.js +1 -4
  135. package/dist-lib/schema/helpers.d.ts +4 -0
  136. package/dist-lib/schema/helpers.js +20 -0
  137. package/dist-lib/schema/local.d.ts +1 -1
  138. package/dist-lib/schema/local.js +70 -2
  139. package/dist-lib/schema/qvac.js +30 -0
  140. package/dist-lib/spec/bind.js +20 -3
  141. package/dist-lib/spec/calls.js +19 -15
  142. package/dist-lib/spec/capabilities.d.ts +42 -34
  143. package/dist-lib/spec/hrpc/hrpc.json +82 -30
  144. package/dist-lib/spec/hrpc/index.js +106 -57
  145. package/dist-lib/spec/hrpc/messages.js +5420 -5019
  146. package/dist-lib/spec/hyperschema/index.js +5424 -5023
  147. package/dist-lib/spec/hyperschema/schema.json +3202 -2666
  148. package/dist-lib/spec/hyperschema/types.d.ts +455 -319
  149. package/dist-lib/spec/local/hyperdb/db.json +119 -1
  150. package/dist-lib/spec/local/hyperdb/index.js +414 -2
  151. package/dist-lib/spec/local/hyperdb/messages.js +505 -124
  152. package/dist-lib/spec/local/hyperschema/index.js +203 -5
  153. package/dist-lib/spec/local/hyperschema/schema.json +136 -1
  154. package/dist-lib/spec/local/hyperschema/types.d.ts +32 -1
  155. package/dist-lib/spec/mesh/hyperdb/db.json +55 -3
  156. package/dist-lib/spec/mesh/hyperdb/index.js +119 -3
  157. package/dist-lib/spec/mesh/hyperdb/messages.js +244 -172
  158. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
  159. package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
  160. package/dist-lib/spec/mesh/hyperdispatch/messages.js +74 -2
  161. package/dist-lib/spec/mesh/hyperschema/index.js +74 -2
  162. package/dist-lib/spec/mesh/hyperschema/schema.json +27 -1
  163. package/dist-lib/spec/mesh/hyperschema/types.d.ts +9 -1
  164. package/package.json +15 -9
  165. package/tui/app.mjs +376 -151
  166. package/tui/boot.mjs +2 -1
  167. package/tui/lib/busy-words.mjs +2 -1
  168. package/tui/lib/knowledge-command.mjs +11 -10
  169. package/tui/lib/knowledge-import.mjs +7 -5
  170. package/tui/lib/render.mjs +4 -3
  171. package/tui/lib/usage.mjs +2 -1
  172. package/tui/main.mjs +2 -1
  173. package/tui/lib/projection-env.mjs +0 -24
@@ -40,14 +40,23 @@ export class Files extends EventEmitter {
40
40
  this._store = new BlobStore(mesh);
41
41
  this._store.on('update', this._updated);
42
42
  }
43
- read(reference, options = {}) {
44
- return this._store.read(reference, options);
43
+ async *read(reference, options = {}) {
44
+ let recorded = false;
45
+ for await (const frame of this._store.read(reference, options)) {
46
+ if (!recorded)
47
+ this._mesh.fileIndex?.read(reference);
48
+ recorded = true;
49
+ yield frame;
50
+ }
45
51
  }
46
- readBytes(reference, options = {}) {
47
- return this._store.readBytes(reference, options);
52
+ async readBytes(reference, options = {}) {
53
+ const bytes = await this._store.readBytes(reference, options);
54
+ this._mesh.fileIndex?.read(reference);
55
+ return bytes;
48
56
  }
49
- download(reference, options) {
50
- return this._store.download(reference, options);
57
+ async download(reference, options) {
58
+ await this._store.download(reference, options);
59
+ this._mesh.fileIndex?.read(reference);
51
60
  }
52
61
  write(chunks, options = {}) {
53
62
  return this._store.write(chunks, options);
@@ -132,8 +141,9 @@ export class Files extends EventEmitter {
132
141
  hasLiveSource(chatId, reference) {
133
142
  return hasLiveSource(this._mesh, chatId, reference);
134
143
  }
135
- syncAttachment(chunk) {
136
- return syncAttachmentChunk(this._mesh, chunk);
144
+ async syncAttachment(chunk) {
145
+ await this._mesh.fileIndex?.refresh([chunk]);
146
+ await syncAttachmentChunk(this._mesh, chunk);
137
147
  }
138
148
  syncKnowledge(row) {
139
149
  return syncKnowledgeTombstone(this._mesh, row);
@@ -0,0 +1,16 @@
1
+ import type { QvacChunk } from '../../../spec/mesh/hyperschema/types.d.ts';
2
+ import type { FileKey, KnownFile } from '../local/files.ts';
3
+ export type FileChunk = Pick<QvacChunk, 'type' | 'chatId' | 'attachment'>;
4
+ export interface MeshFileIndex {
5
+ record(chunks: ReadonlyArray<FileChunk>): Promise<void>;
6
+ refresh(chunks: ReadonlyArray<FileChunk>): Promise<void>;
7
+ read(reference: string): void;
8
+ markCleared(blobId: string): Promise<void>;
9
+ }
10
+ export interface FileOwner {
11
+ readonly meshKey: Buffer;
12
+ readonly ownCoreKey: Buffer;
13
+ }
14
+ export declare function knownFile(chunk: FileChunk, owner: FileOwner): KnownFile | null;
15
+ export declare function referencedFile(reference: string, owner: FileOwner): KnownFile | null;
16
+ export declare function fileKeyOf(reference: string, { meshKey, ownCoreKey }: FileOwner): FileKey | null;
@@ -0,0 +1,38 @@
1
+ import b4a from 'b4a';
2
+ import { parseBlobReference } from './store/reference.js';
3
+ export function knownFile(chunk, owner) {
4
+ const attachment = chunk.type === 'attachment' ? chunk.attachment : null;
5
+ if (!attachment)
6
+ return null;
7
+ const file = referencedFile(attachment.blobId, owner);
8
+ if (!file)
9
+ return null;
10
+ return {
11
+ ...file,
12
+ chatId: chunk.chatId,
13
+ attachmentId: attachment.id,
14
+ fileName: attachment.fileName,
15
+ mimeType: attachment.mimeType,
16
+ byteLength: attachment.byteLength,
17
+ deletedAt: attachment.deletedAt || undefined
18
+ };
19
+ }
20
+ export function referencedFile(reference, owner) {
21
+ const key = fileKeyOf(reference, owner);
22
+ if (!key)
23
+ return null;
24
+ return { ...key, blobId: reference, own: b4a.equals(key.coreKey, owner.ownCoreKey) };
25
+ }
26
+ export function fileKeyOf(reference, { meshKey, ownCoreKey }) {
27
+ let parsed;
28
+ try {
29
+ parsed = parseBlobReference(reference);
30
+ }
31
+ catch {
32
+ return null;
33
+ }
34
+ if (parsed.inline)
35
+ return null;
36
+ const coreKey = parsed.coreKey ? Buffer.from(parsed.coreKey, 'hex') : ownCoreKey;
37
+ return { meshKey, coreKey, blockOffset: parsed.blobId.blockOffset };
38
+ }
@@ -3,6 +3,7 @@ import { IndexingClaims } from './claims.js';
3
3
  import { ArtifactOwnership } from './ownership.js';
4
4
  import { listIndexingByKnowledge, resolveIndexing } from './protocol.js';
5
5
  import { fileRecord } from '../records/index.js';
6
+ import b4a from 'b4a';
6
7
  export class FileDocument {
7
8
  _mesh;
8
9
  record;
@@ -45,7 +46,7 @@ export class FileDocument {
45
46
  this.ownership = new ArtifactOwnership(mesh, state);
46
47
  }
47
48
  get localCoreKey() {
48
- return this._mesh.blobs.core.key.toString('hex');
49
+ return b4a.toHex(this._mesh.blobs.core.key);
49
50
  }
50
51
  artifact(id) {
51
52
  return this._mesh.files.processorState(this._processor.id).artifact(id);
@@ -1,4 +1,5 @@
1
1
  import { compareBlobReferences } from '../store/index.js';
2
+ import b4a from 'b4a';
2
3
  export class ArtifactOwnership {
3
4
  _mesh;
4
5
  _state;
@@ -7,7 +8,8 @@ export class ArtifactOwnership {
7
8
  this._state = state;
8
9
  }
9
10
  async onPublished(artifactId, blobRef, artifactBlobId, landedBlobId) {
10
- if (compareBlobReferences(landedBlobId, artifactBlobId, this._mesh.blobs.core.key.toString('hex')) === false) {
11
+ if (compareBlobReferences(landedBlobId, artifactBlobId, b4a.toHex(this._mesh.blobs.core.key)) ===
12
+ false) {
11
13
  await this._mesh.files.clear(this._mesh.files.reference(blobRef), artifactId).catch(noop);
12
14
  return;
13
15
  }
@@ -17,7 +19,7 @@ export class ArtifactOwnership {
17
19
  const local = this._state.artifact(artifactId);
18
20
  const mine = local.published;
19
21
  if (!mine ||
20
- compareBlobReferences(mine.artifactBlobId, winnerBlobId, this._mesh.blobs.core.key.toString('hex')) !== false) {
22
+ compareBlobReferences(mine.artifactBlobId, winnerBlobId, b4a.toHex(this._mesh.blobs.core.key)) !== false) {
21
23
  return;
22
24
  }
23
25
  local.published = undefined;
@@ -28,7 +30,8 @@ export class ArtifactOwnership {
28
30
  const row = await this._mesh.files.artifact(artifactId);
29
31
  const mine = this._mesh.files.reference(blobRef);
30
32
  if (row &&
31
- compareBlobReferences(row.artifactBlobId, mine, this._mesh.blobs.core.key.toString('hex')) !== false) {
33
+ compareBlobReferences(row.artifactBlobId, mine, b4a.toHex(this._mesh.blobs.core.key)) !==
34
+ false) {
32
35
  this._state.artifact(artifactId).published = { artifactBlobId: mine };
33
36
  return;
34
37
  }
@@ -9,4 +9,5 @@ export async function syncAttachmentChunk(mesh, chunk) {
9
9
  return;
10
10
  logger.debug('sync', 'attachment tombstone observed — reclaiming blob bytes', { id: chunk.id });
11
11
  await mesh.files.clear(attachment.blobId, chunk.id);
12
+ await mesh.fileIndex?.markCleared(attachment.blobId);
12
13
  }
@@ -9,6 +9,7 @@ export async function syncKnowledgeTombstone(mesh, row) {
9
9
  logger.debug('sync', 'knowledge tombstone observed - reclaiming blob bytes', { id: row.id });
10
10
  if (typeof row.blobId === 'string' && !(await hasLiveSource(mesh, row.chatId, row.blobId))) {
11
11
  await mesh.files.clear(row.blobId, row.id);
12
+ await mesh.fileIndex?.markCleared(row.blobId);
12
13
  }
13
14
  for (const artifact of await listKnowledgeArtifactsByKnowledge(mesh, { knowledgeId: row.id })) {
14
15
  if (mesh.closing)
@@ -5,6 +5,7 @@ import { getChat, setChatRagEnabled } from '../../chat.js';
5
5
  import { listChunks } from '../../chunk.js';
6
6
  import { IMAGE_MIMES } from '../../image/normalize.js';
7
7
  import { addKnowledge, listKnowledge, removeKnowledge } from './knowledge.js';
8
+ import b4a from 'b4a';
8
9
  export function isKnowledgeRoutedAttachment(input) {
9
10
  if (IMAGE_MIMES.has(input.mimeType.toLowerCase()))
10
11
  return false;
@@ -31,7 +32,7 @@ export async function routeAttachmentToKnowledge(mesh, chatId, attachment) {
31
32
  return knowledge;
32
33
  }
33
34
  export async function hasLiveSource(mesh, chatId, reference) {
34
- const same = (blobId) => compareBlobReferences(blobId, reference, mesh.blobs.core.key.toString('hex')) !== false;
35
+ const same = (blobId) => compareBlobReferences(blobId, reference, b4a.toHex(mesh.blobs.core.key)) !== false;
35
36
  if ((await listKnowledge(mesh, { chatId })).some((row) => same(row.blobId)))
36
37
  return true;
37
38
  return (await listChunks(mesh, { chatId, type: 'attachment' })).some(({ attachment }) => attachment && !attachment.deletedAt && same(attachment.blobId));
@@ -26,4 +26,4 @@ export declare function readBlob(blobs: Hyperblobs, blobId: BlobId, options?: {
26
26
  bytesRead: number;
27
27
  byteLength: number;
28
28
  }>;
29
- export declare function clearBlob(mesh: BlobHost, blobId: string, id: string): Promise<void>;
29
+ export declare function clearBlob(mesh: BlobHost, blobId: string, id: string): Promise<boolean>;
@@ -1,6 +1,7 @@
1
1
  import { AbortedError } from '@qvac/harness/race-abort';
2
2
  import { nextFrame } from '../../next-frame.js';
3
3
  import { parseBlobReference } from './reference.js';
4
+ import b4a from 'b4a';
4
5
  export async function* writeBlob(blobs, chunks, options = {}) {
5
6
  const ws = blobs.createWriteStream();
6
7
  const done = new Promise((resolve, reject) => {
@@ -51,7 +52,7 @@ export async function* writeBlob(blobs, chunks, options = {}) {
51
52
  }
52
53
  const noop = () => { };
53
54
  export function stringifyBlobId(blobs, id) {
54
- return JSON.stringify({ ...id, coreKey: blobs.core.key.toString('hex') });
55
+ return JSON.stringify({ ...id, coreKey: b4a.toHex(blobs.core.key) });
55
56
  }
56
57
  export async function* readBlob(blobs, blobId, options = {}) {
57
58
  const { signal, ...readOptions } = options;
@@ -75,16 +76,18 @@ export async function* readBlob(blobs, blobId, options = {}) {
75
76
  export async function clearBlob(mesh, blobId, id) {
76
77
  try {
77
78
  if (parseBlobReference(blobId).inline)
78
- return;
79
+ return true;
79
80
  await clearBlobRef(mesh, blobId);
81
+ return true;
80
82
  }
81
83
  catch (err) {
82
84
  logger.warn('sync', 'blob clear failed', { id, err });
85
+ return false;
83
86
  }
84
87
  }
85
88
  async function clearBlobRef(mesh, blobId) {
86
89
  const { coreKey, ...id } = JSON.parse(blobId);
87
- if (!coreKey || coreKey === mesh.blobs.core.key.toString('hex')) {
90
+ if (!coreKey || coreKey === b4a.toHex(mesh.blobs.core.key)) {
88
91
  await mesh.blobs.clear(id);
89
92
  return;
90
93
  }
@@ -30,7 +30,7 @@ export declare class BlobStore extends EventEmitter {
30
30
  }, any, any>;
31
31
  readBytes(reference: string, options?: Parameters<typeof readBlobReferenceBytes>[2]): Promise<Buffer<ArrayBufferLike>>;
32
32
  download(reference: string, options: Parameters<typeof downloadBlobReference>[2]): Promise<void>;
33
- clear(reference: string, id: string): Promise<void>;
33
+ clear(reference: string, id: string): Promise<boolean>;
34
34
  available(reference: string): Promise<boolean>;
35
35
  sweep(references: string[], coreLength: number): Promise<void>;
36
36
  close(): Promise<void>;
@@ -1,6 +1,7 @@
1
1
  import { readBlob } from './blob.js';
2
2
  import { AbortedError, raceAbort } from '@qvac/harness/race-abort';
3
3
  import { MAX_REFERENCE_CHARS, readInline } from './inline.js';
4
+ import b4a from 'b4a';
4
5
  const CORE_KEY_PATTERN = /^[0-9a-f]{64}$/;
5
6
  const DEFAULT_REMOTE_TIMEOUT_MS = 15_000;
6
7
  const INVALID_BLOB_REFERENCE = 'invalid blob reference';
@@ -113,8 +114,8 @@ export async function* readBlobReference(mesh, blobReference, options = {}) {
113
114
  yield { data, bytesRead: data.length, byteLength: data.length };
114
115
  return;
115
116
  }
116
- const local = !coreKey || coreKey === mesh.blobs.core.key.toString('hex');
117
- const readOptions = local || options.wait === false ? { wait: false } : { timeout: remoteTimeout(options) };
117
+ const local = !coreKey || coreKey === b4a.toHex(mesh.blobs.core.key);
118
+ const readOptions = options.wait === false ? { wait: false } : { timeout: remoteTimeout(options) };
118
119
  let remote = null;
119
120
  try {
120
121
  if (!local)
@@ -139,7 +140,7 @@ export async function downloadBlobReference(mesh, reference, { signal }) {
139
140
  return;
140
141
  let remote = null;
141
142
  try {
142
- if (coreKey && coreKey !== mesh.blobs.core.key.toString('hex')) {
143
+ if (coreKey && coreKey !== b4a.toHex(mesh.blobs.core.key)) {
143
144
  remote = await mesh.openBlobs(Buffer.from(coreKey, 'hex'));
144
145
  }
145
146
  if (signal.aborted)
@@ -3,22 +3,26 @@ import ReadyResource from 'ready-resource';
3
3
  import type { LinkableAttachment } from '../../attachment-link.ts';
4
4
  import type Hyperblobs from 'hyperblobs';
5
5
  export type MeshBlobKeys = {
6
+ key: Buffer;
6
7
  blobs: Hyperblobs;
7
8
  encryptionKey: Buffer;
8
9
  };
9
- export type MeshForBlobCore = (coreKey: string | null) => MeshBlobKeys | null;
10
+ export type MeshForBlobCore = (coreKey: string, blockOffset: number | undefined, meshKey: Buffer | undefined) => MeshBlobKeys | null | Promise<MeshBlobKeys | null>;
11
+ export type BlobRead = (mesh: MeshBlobKeys, coreKey: Buffer, blockOffset: number) => void;
10
12
  export declare class CoreBlobServer extends ReadyResource {
11
13
  private readonly _store;
12
14
  private readonly _mesh;
13
15
  private readonly _server;
14
16
  private readonly _probes;
15
17
  private readonly _readTimeoutMs;
18
+ private readonly _onRead;
16
19
  private _suspended;
17
- constructor(store: Corestore, mesh: MeshForBlobCore, { readTimeoutMs }?: {
18
- readTimeoutMs?: number | undefined;
20
+ constructor(store: Corestore, mesh: MeshForBlobCore, { readTimeoutMs, onRead }?: {
21
+ readTimeoutMs?: number;
22
+ onRead?: BlobRead | null;
19
23
  });
20
24
  get port(): number;
21
- link({ blobId, mimeType }: LinkableAttachment, source?: MeshBlobKeys | null): string | null;
25
+ link({ blobId, mimeType }: LinkableAttachment, source: MeshBlobKeys): string | null;
22
26
  suspend(): Promise<void>;
23
27
  resume(): Promise<void>;
24
28
  _open(): Promise<void>;
@@ -1,18 +1,21 @@
1
1
  import BlobServer, {} from 'hypercore-blob-server';
2
2
  import ReadyResource from 'ready-resource';
3
3
  import { isInvalidBlobReferenceError, parseBlobReference } from './reference.js';
4
+ import b4a from 'b4a';
4
5
  export class CoreBlobServer extends ReadyResource {
5
6
  _store;
6
7
  _mesh;
7
8
  _server;
8
9
  _probes = new Set();
9
10
  _readTimeoutMs;
11
+ _onRead;
10
12
  _suspended = false;
11
- constructor(store, mesh, { readTimeoutMs = 15_000 } = {}) {
13
+ constructor(store, mesh, { readTimeoutMs = 15_000, onRead = null } = {}) {
12
14
  super();
13
15
  this._store = store.session();
14
16
  this._mesh = mesh;
15
17
  this._readTimeoutMs = readTimeoutMs;
18
+ this._onRead = onRead;
16
19
  this._server = new BlobServer(this._store, {
17
20
  resolve: (key, info) => this._resolve(key, info)
18
21
  });
@@ -20,16 +23,15 @@ export class CoreBlobServer extends ReadyResource {
20
23
  get port() {
21
24
  return this._server.port;
22
25
  }
23
- link({ blobId, mimeType }, source = this._mesh(null)) {
26
+ link({ blobId, mimeType }, source) {
24
27
  if (!this.opened || this.closing)
25
28
  return null;
26
- if (!source)
27
- return null;
28
29
  const parsed = this._parse(blobId);
29
30
  if (!parsed || parsed.inline)
30
31
  return null;
31
- return this._server.getLink(parsed.coreKey ?? source.blobs.core.key.toString('hex'), {
32
+ return this._server.getLink(parsed.coreKey ?? b4a.toHex(source.blobs.core.key), {
32
33
  blob: parsed.blobId,
34
+ drive: source.key,
33
35
  type: mimeType || 'application/octet-stream'
34
36
  });
35
37
  }
@@ -64,11 +66,12 @@ export class CoreBlobServer extends ReadyResource {
64
66
  await this._store.close();
65
67
  }
66
68
  async _resolve(key, info) {
67
- const current = this._mesh(key.toString('hex'));
69
+ const { blob, range } = info;
70
+ const meshKey = info.drive ?? undefined;
71
+ const current = await this._mesh(b4a.toHex(key), blob?.blockOffset, meshKey);
68
72
  if (!current || this.closing || this._suspended)
69
73
  return null;
70
74
  const keys = { encryptionKey: current.encryptionKey };
71
- const { blob, range } = info;
72
75
  if (!blob || info.head || blob.byteLength === 0)
73
76
  return keys;
74
77
  const start = range?.start ?? 0;
@@ -90,12 +93,13 @@ export class CoreBlobServer extends ReadyResource {
90
93
  if (!position || this.closing || this._suspended || Date.now() >= deadline)
91
94
  return null;
92
95
  const bytes = await core.get(position[0], { timeout: Math.max(1, deadline - Date.now()) });
93
- return bytes &&
94
- this._mesh(key.toString('hex')) === current &&
96
+ const served = bytes &&
97
+ (await this._mesh(b4a.toHex(key), blob.blockOffset, meshKey)) === current &&
95
98
  !this.closing &&
96
- !this._suspended
97
- ? keys
98
- : null;
99
+ !this._suspended;
100
+ if (served)
101
+ this._onRead?.(current, key, blob.blockOffset);
102
+ return served ? keys : null;
99
103
  }
100
104
  catch {
101
105
  return null;
@@ -1,7 +1,8 @@
1
1
  import { parseBlobReference, parseBlobReferenceCoreKey } from './reference.js';
2
+ import b4a from 'b4a';
2
3
  export async function sweepUnreferenced(blobs, references, coreLength) {
3
4
  const ranges = [];
4
- const key = blobs.core.key.toString('hex');
5
+ const key = b4a.toHex(blobs.core.key);
5
6
  for (const reference of references)
6
7
  addLocalRange(ranges, reference, key, coreLength);
7
8
  await clearGaps(blobs, mergeRanges(ranges), coreLength);
@@ -7,8 +7,8 @@ export declare function uploadFile(mesh: Mesh, frames: AsyncIterable<AssistantUp
7
7
  deviceId?: Buffer | null;
8
8
  inactivityMs?: number;
9
9
  }): AsyncGenerator<{
10
- bytesWritten: number;
11
10
  attachment?: undefined;
11
+ bytesWritten: number;
12
12
  knowledge?: undefined;
13
13
  } | {
14
14
  bytesWritten: number;
@@ -5,7 +5,7 @@ import { agentSkillValues } from './skill-secrets.js';
5
5
  const SCOPE = ['p2p', 'skill-values'];
6
6
  export const LEGACY_HELD_AGENTS_KEY = 'skill-values.held-agents';
7
7
  export function heldAgentsKey(meshKey) {
8
- return `${LEGACY_HELD_AGENTS_KEY}.${meshKey.toString('hex')}`;
8
+ return `${LEGACY_HELD_AGENTS_KEY}.${b4a.toHex(meshKey)}`;
9
9
  }
10
10
  export class HostedAgents {
11
11
  _mesh;
@@ -4,6 +4,7 @@ export declare function imageCacheName(image: {
4
4
  blobId: string;
5
5
  fileName: string;
6
6
  }): string;
7
+ export declare function imageCacheKey(blobId: string): string;
7
8
  export declare function sweepImageCache(dir: string, { maxBytes, maxAgeMs, keep, inUse }?: {
8
9
  maxBytes?: number;
9
10
  maxAgeMs?: number;
@@ -1,14 +1,18 @@
1
1
  import fs from 'fs';
2
2
  import crypto from 'hypercore-crypto';
3
3
  import path from 'path';
4
+ import b4a from 'b4a';
4
5
  export const IMAGE_CACHE_MAX_BYTES = 512 * 1024 * 1024;
5
6
  export const IMAGE_CACHE_MAX_AGE_MS = 365 * 24 * 60 * 60 * 1000;
6
7
  const STALE_PARTIAL_MS = 60 * 60 * 1000;
7
8
  const STAGING_SUFFIX = /\.partial-[0-9a-f]{8}$/;
9
+ const CACHE_KEY_CHARS = 32;
8
10
  export function imageCacheName(image) {
9
- const key = crypto.hash(Buffer.from(image.blobId)).toString('hex').slice(0, 32);
10
11
  const name = image.fileName.replace(/[^a-zA-Z0-9_.-]/g, '_').slice(-40);
11
- return `${key}-${name}`;
12
+ return `${imageCacheKey(image.blobId)}-${name}`;
13
+ }
14
+ export function imageCacheKey(blobId) {
15
+ return b4a.toHex(crypto.hash(Buffer.from(blobId))).slice(0, CACHE_KEY_CHARS);
12
16
  }
13
17
  export async function sweepImageCache(dir, { maxBytes = IMAGE_CACHE_MAX_BYTES, maxAgeMs = IMAGE_CACHE_MAX_AGE_MS, keep, inUse } = {}) {
14
18
  const names = await fs.promises.readdir(dir).catch(toNull);
@@ -21,7 +25,7 @@ export async function sweepImageCache(dir, { maxBytes = IMAGE_CACHE_MAX_BYTES, m
21
25
  total += entry.size;
22
26
  if (inUse?.(entry.name))
23
27
  continue;
24
- if (keep && !keep.has(entry.name)) {
28
+ if (keep && !keep.has(entry.name.slice(0, CACHE_KEY_CHARS))) {
25
29
  await fs.promises.unlink(entry.file).catch(noop);
26
30
  total -= entry.size;
27
31
  continue;
@@ -0,0 +1,24 @@
1
+ import type HyperDB from 'hyperdb';
2
+ import type { LocalFile } from '../../../spec/local/hyperschema/types.d.ts';
3
+ export type { LocalFile };
4
+ export type KnownFile = Omit<LocalFile, 'lastAccessed' | 'cleared'>;
5
+ export type FileKey = Pick<LocalFile, 'meshKey' | 'coreKey' | 'blockOffset'>;
6
+ type Locked = <T>(fn: () => Promise<T>) => Promise<T>;
7
+ export declare class LocalFiles {
8
+ private readonly _db;
9
+ private readonly _locked;
10
+ constructor(db: HyperDB, locked: Locked);
11
+ record(files: ReadonlyArray<KnownFile>, now?: number): Promise<void>;
12
+ refresh(files: ReadonlyArray<KnownFile>): Promise<void>;
13
+ read(file: KnownFile, now?: number): Promise<void>;
14
+ touch(key: FileKey, now?: number): Promise<void>;
15
+ private _write;
16
+ markCleared(file: FileKey): Promise<void>;
17
+ meshOf(coreKey: Buffer, blockOffset: number): Promise<Buffer | null>;
18
+ backfilled(meshKey: Buffer): Promise<boolean>;
19
+ markBackfilled(meshKey: Buffer): Promise<void>;
20
+ list(meshKey?: Buffer): Promise<LocalFile[]>;
21
+ live(meshKey?: Buffer): Promise<LocalFile[]>;
22
+ stale(cutoff: number): Promise<LocalFile[]>;
23
+ }
24
+ export declare function dropMeshFiles(tx: HyperDB, meshKey: Buffer): Promise<void>;
@@ -0,0 +1,154 @@
1
+ import b4a from 'b4a';
2
+ const FILES = '@local/files';
3
+ const FILES_BY_BLOB = '@local/files-by-blob';
4
+ const RECLAIMABLE_FILES = '@local/reclaimable-files-by-last-accessed';
5
+ const LIVE_FILES = '@local/live-files';
6
+ const CONFIG = '@local/config';
7
+ const TOUCH_GRAIN_MS = 60 * 60 * 1000;
8
+ export class LocalFiles {
9
+ _db;
10
+ _locked;
11
+ constructor(db, locked) {
12
+ this._db = db;
13
+ this._locked = locked;
14
+ }
15
+ record(files, now = Date.now()) {
16
+ return this._write(files, (file, known) => {
17
+ if (known && described(known, file)) {
18
+ return null;
19
+ }
20
+ return {
21
+ ...merged(known, file),
22
+ own: file.own || (known?.own ?? false),
23
+ lastAccessed: known?.lastAccessed ?? now,
24
+ cleared: known?.cleared ?? false
25
+ };
26
+ });
27
+ }
28
+ refresh(files) {
29
+ return this._write(files, (file, known) => {
30
+ if (!known || described(known, file)) {
31
+ return null;
32
+ }
33
+ return { ...known, ...merged(known, file) };
34
+ });
35
+ }
36
+ read(file, now = Date.now()) {
37
+ return this._write([file], (_, known) => {
38
+ if (known && readRecently(known, now)) {
39
+ return null;
40
+ }
41
+ return {
42
+ ...merged(known, file),
43
+ own: file.own || (known?.own ?? false),
44
+ lastAccessed: now,
45
+ cleared: false
46
+ };
47
+ });
48
+ }
49
+ touch(key, now = Date.now()) {
50
+ return this._write([key], (_, known) => {
51
+ if (!known || readRecently(known, now)) {
52
+ return null;
53
+ }
54
+ return { ...known, lastAccessed: now, cleared: false };
55
+ });
56
+ }
57
+ _write(files, next) {
58
+ return this._locked(async () => {
59
+ const tx = this._db.transaction();
60
+ let changed = false;
61
+ for (const file of files) {
62
+ const known = await tx.get(FILES, keyOf(file));
63
+ const row = next(file, known);
64
+ if (!row) {
65
+ continue;
66
+ }
67
+ await tx.insert(FILES, row);
68
+ changed = true;
69
+ }
70
+ if (changed) {
71
+ await tx.flush();
72
+ }
73
+ else {
74
+ await tx.close();
75
+ }
76
+ });
77
+ }
78
+ markCleared(file) {
79
+ return this._locked(async () => {
80
+ const tx = this._db.transaction();
81
+ const known = await tx.get(FILES, keyOf(file));
82
+ if (!known) {
83
+ await tx.close();
84
+ return;
85
+ }
86
+ await tx.insert(FILES, { ...known, cleared: true });
87
+ await tx.flush();
88
+ });
89
+ }
90
+ async meshOf(coreKey, blockOffset) {
91
+ const key = { coreKey, blockOffset };
92
+ const row = await this._db.findOne(FILES_BY_BLOB, { gte: key, lte: key });
93
+ return row?.meshKey ?? null;
94
+ }
95
+ async backfilled(meshKey) {
96
+ return (await this._db.get(CONFIG, { key: backfillKey(meshKey) })) !== null;
97
+ }
98
+ markBackfilled(meshKey) {
99
+ return this._locked(async () => {
100
+ const tx = this._db.transaction();
101
+ await tx.insert(CONFIG, { key: backfillKey(meshKey), value: true });
102
+ await tx.flush();
103
+ });
104
+ }
105
+ list(meshKey) {
106
+ return this._db.find(FILES, meshRange(meshKey)).toArray();
107
+ }
108
+ live(meshKey) {
109
+ return this._db.find(LIVE_FILES, meshRange(meshKey)).toArray();
110
+ }
111
+ stale(cutoff) {
112
+ return this._db.find(RECLAIMABLE_FILES, { lt: { lastAccessed: cutoff } }).toArray();
113
+ }
114
+ }
115
+ export async function dropMeshFiles(tx, meshKey) {
116
+ const rows = await tx.find(FILES, { gte: { meshKey }, lte: { meshKey } }).toArray();
117
+ for (const row of rows)
118
+ await tx.delete(FILES, keyOf(row));
119
+ await tx.delete(CONFIG, { key: backfillKey(meshKey) });
120
+ }
121
+ function meshRange(meshKey) {
122
+ return meshKey ? { gte: { meshKey }, lte: { meshKey } } : {};
123
+ }
124
+ function backfillKey(meshKey) {
125
+ return `files/backfilled/${b4a.toHex(meshKey)}`;
126
+ }
127
+ function readRecently(known, now) {
128
+ return !known.cleared && now - known.lastAccessed < TOUCH_GRAIN_MS;
129
+ }
130
+ function keyOf(file) {
131
+ return { meshKey: file.meshKey, coreKey: file.coreKey, blockOffset: file.blockOffset };
132
+ }
133
+ function merged(known, file) {
134
+ return {
135
+ ...file,
136
+ chatId: file.chatId ?? known?.chatId,
137
+ attachmentId: file.attachmentId ?? known?.attachmentId,
138
+ fileName: file.fileName ?? known?.fileName,
139
+ mimeType: file.mimeType ?? known?.mimeType,
140
+ byteLength: file.byteLength ?? known?.byteLength,
141
+ deletedAt: file.deletedAt ?? known?.deletedAt
142
+ };
143
+ }
144
+ function described(known, file) {
145
+ const next = merged(known, file);
146
+ return ((!file.own || known.own === true) &&
147
+ next.blobId === known.blobId &&
148
+ next.chatId === known.chatId &&
149
+ next.attachmentId === known.attachmentId &&
150
+ next.fileName === known.fileName &&
151
+ next.mimeType === known.mimeType &&
152
+ next.byteLength === known.byteLength &&
153
+ next.deletedAt === known.deletedAt);
154
+ }