@qvac/core 0.11.0 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CHANGELOG.md +148 -1
  2. package/README.md +19 -16
  3. package/dist-lib/index.d.ts +1 -0
  4. package/dist-lib/index.js +1 -0
  5. package/dist-lib/lib/api/chunk-delta.d.ts +6 -2
  6. package/dist-lib/lib/api/chunk-delta.js +69 -7
  7. package/dist-lib/lib/api/chunks.js +10 -4
  8. package/dist-lib/lib/api/config.d.ts +4 -1
  9. package/dist-lib/lib/api/config.js +23 -4
  10. package/dist-lib/lib/api/devices.js +3 -1
  11. package/dist-lib/lib/api/rag.js +11 -7
  12. package/dist-lib/lib/assistant/harness.d.ts +3 -1
  13. package/dist-lib/lib/assistant/harness.js +10 -0
  14. package/dist-lib/lib/assistant/history.d.ts +1 -0
  15. package/dist-lib/lib/assistant/history.js +9 -3
  16. package/dist-lib/lib/assistant/message.js +2 -2
  17. package/dist-lib/lib/assistant-tree.d.ts +2 -0
  18. package/dist-lib/lib/assistant-tree.js +5 -3
  19. package/dist-lib/lib/assistant.d.ts +6 -1
  20. package/dist-lib/lib/assistant.js +77 -24
  21. package/dist-lib/lib/client.js +10 -1
  22. package/dist-lib/lib/compaction.d.ts +2 -1
  23. package/dist-lib/lib/compaction.js +14 -4
  24. package/dist-lib/lib/constants.d.ts +3 -0
  25. package/dist-lib/lib/constants.js +3 -0
  26. package/dist-lib/lib/core/archived-mesh.js +1 -0
  27. package/dist-lib/lib/core/chat-activity.js +12 -15
  28. package/dist-lib/lib/core/chunk.d.ts +14 -3
  29. package/dist-lib/lib/core/chunk.js +112 -22
  30. package/dist-lib/lib/core/collections.d.ts +1 -0
  31. package/dist-lib/lib/core/collections.js +1 -0
  32. package/dist-lib/lib/core/config-value.d.ts +6 -0
  33. package/dist-lib/lib/core/config-value.js +14 -0
  34. package/dist-lib/lib/core/contract/chat.d.ts +1 -0
  35. package/dist-lib/lib/core/contract/chat.js +8 -1
  36. package/dist-lib/lib/core/contract/chunks.js +20 -6
  37. package/dist-lib/lib/core/contract/compute.d.ts +1 -1
  38. package/dist-lib/lib/core/contract/compute.js +2 -1
  39. package/dist-lib/lib/core/contract/devices.js +3 -2
  40. package/dist-lib/lib/core/departures.js +1 -0
  41. package/dist-lib/lib/core/devices.d.ts +2 -1
  42. package/dist-lib/lib/core/devices.js +4 -2
  43. package/dist-lib/lib/core/files/attachments.js +3 -3
  44. package/dist-lib/lib/core/files/index.d.ts +1 -1
  45. package/dist-lib/lib/core/files/index.js +3 -2
  46. package/dist-lib/lib/core/files/processing/index.d.ts +1 -0
  47. package/dist-lib/lib/core/files/processing/index.js +8 -0
  48. package/dist-lib/lib/core/files/processing/settle.d.ts +1 -0
  49. package/dist-lib/lib/core/files/processing/settle.js +20 -13
  50. package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
  51. package/dist-lib/lib/core/files/rag/index.js +1 -1
  52. package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +1 -0
  53. package/dist-lib/lib/core/files/rag/retrieval/full-text.js +4 -1
  54. package/dist-lib/lib/core/hosted-agents.d.ts +3 -0
  55. package/dist-lib/lib/core/hosted-agents.js +19 -0
  56. package/dist-lib/lib/core/inference-calls-receiver.js +2 -2
  57. package/dist-lib/lib/core/known-sender.d.ts +3 -0
  58. package/dist-lib/lib/core/known-sender.js +26 -0
  59. package/dist-lib/lib/core/local/index.d.ts +2 -0
  60. package/dist-lib/lib/core/local/index.js +17 -10
  61. package/dist-lib/lib/core/mesh.d.ts +5 -1
  62. package/dist-lib/lib/core/mesh.js +6 -6
  63. package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
  64. package/dist-lib/lib/core/migrations/chat-chunks.js +43 -0
  65. package/dist-lib/lib/core/notification.js +16 -13
  66. package/dist-lib/lib/core/skill-overlay-receiver.js +2 -2
  67. package/dist-lib/lib/core/skill-secrets.js +2 -2
  68. package/dist-lib/lib/core.d.ts +17 -1
  69. package/dist-lib/lib/core.js +55 -5
  70. package/dist-lib/lib/credential-pusher.d.ts +11 -7
  71. package/dist-lib/lib/credential-pusher.js +40 -8
  72. package/dist-lib/lib/device-tree.d.ts +18 -0
  73. package/dist-lib/lib/device-tree.js +32 -0
  74. package/dist-lib/lib/integrations/setup-routes.js +7 -5
  75. package/dist-lib/lib/kv-cache-retention.d.ts +34 -0
  76. package/dist-lib/lib/kv-cache-retention.js +98 -0
  77. package/dist-lib/schema/content.js +2 -1
  78. package/dist-lib/schema/engine.js +20 -3
  79. package/dist-lib/schema/field-order.js +8 -1
  80. package/dist-lib/schema/local.js +7 -3
  81. package/dist-lib/schema/qvac.js +23 -4
  82. package/dist-lib/spec/hrpc/messages.js +1240 -1194
  83. package/dist-lib/spec/hyperschema/index.js +1240 -1194
  84. package/dist-lib/spec/hyperschema/schema.json +49 -1
  85. package/dist-lib/spec/hyperschema/types.d.ts +12 -1
  86. package/dist-lib/spec/local/hyperdb/messages.js +70 -20
  87. package/dist-lib/spec/local/hyperschema/index.js +36 -11
  88. package/dist-lib/spec/local/hyperschema/schema.json +21 -1
  89. package/dist-lib/spec/local/hyperschema/types.d.ts +5 -1
  90. package/dist-lib/spec/mesh/hyperdb/db.json +17 -1
  91. package/dist-lib/spec/mesh/hyperdb/index.js +83 -2
  92. package/dist-lib/spec/mesh/hyperdb/messages.js +602 -264
  93. package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +7 -1
  94. package/dist-lib/spec/mesh/hyperdispatch/index.js +18 -1
  95. package/dist-lib/spec/mesh/hyperdispatch/messages.js +137 -14
  96. package/dist-lib/spec/mesh/hyperschema/index.js +137 -14
  97. package/dist-lib/spec/mesh/hyperschema/schema.json +79 -1
  98. package/dist-lib/spec/mesh/hyperschema/types.d.ts +21 -1
  99. package/package.json +13 -9
  100. package/tui/app.mjs +743 -266
  101. package/tui/boot.mjs +2 -2
  102. package/tui/lib/connect-sheet.mjs +350 -0
  103. package/tui/lib/setup.mjs +59 -25
  104. package/tui/lib/skill-connect.mjs +145 -0
@@ -7,7 +7,8 @@ import { linkAttachment } from './attachment-link.js';
7
7
  import { randomId } from './id.js';
8
8
  import { keyRange } from './range.js';
9
9
  import b4a from 'b4a';
10
- import { AGENTS, ATTACHMENTS_BY_MEDIA_KIND, ATTACHMENTS_BY_MEDIA_KIND_ORIGIN, CHUNKS, CHUNKS_BY_CHAT, CHUNKS_BY_TYPE, HISTORY_CHUNKS, RUN_REQUESTS } from './collections.js';
10
+ import { AGENTS, ATTACHMENTS_BY_MEDIA_KIND, ATTACHMENTS_BY_MEDIA_KIND_ORIGIN, CHAT_CHUNKS, CHUNKS, CHUNKS_BY_CHAT, CHUNKS_BY_TYPE, CHUNK_COUNTERS, HISTORY_CHUNKS, RUN_REQUESTS } from './collections.js';
11
+ const TERMINAL_RUN_STATES = new Set(['completed', 'failed', 'canceled']);
11
12
  export const HISTORY_CHUNK_TYPES = [
12
13
  'user-message',
13
14
  'run-request',
@@ -15,12 +16,31 @@ export const HISTORY_CHUNK_TYPES = [
15
16
  'attachment',
16
17
  'compaction'
17
18
  ];
18
- function linkChunks(store, chunks) {
19
- const mint = store.attachmentUrl;
20
- if (!mint) {
21
- return chunks;
19
+ const FALSY_RESULTS = new Map([
20
+ ['false', false],
21
+ ['0', 0],
22
+ ['""', '']
23
+ ]);
24
+ export function readChunk({ toolResult, ...chunk }) {
25
+ if (!toolResult) {
26
+ return { ...chunk, toolResult: null };
22
27
  }
23
- return chunks.map((chunk) => chunk.attachment ? { ...chunk, attachment: linkAttachment(mint, chunk.attachment) } : chunk);
28
+ const stored = chunk.falsyResult ?? '';
29
+ const result = FALSY_RESULTS.has(stored) ? FALSY_RESULTS.get(stored) : (toolResult.result ?? null);
30
+ return { ...chunk, toolResult: { ...toolResult, result } };
31
+ }
32
+ export function linkChunks(store, chunks) {
33
+ const mint = store.attachmentUrl;
34
+ return chunks.map((row) => {
35
+ const chunk = readChunk(row);
36
+ return mint && chunk.attachment
37
+ ? { ...chunk, attachment: linkAttachment(mint, chunk.attachment) }
38
+ : chunk;
39
+ });
40
+ }
41
+ function falsyResultOf(payload) {
42
+ const result = payload.toolResult?.result;
43
+ return result === false || result === 0 || result === '' ? JSON.stringify(result) : undefined;
24
44
  }
25
45
  export async function recordLinks(store, chunks) {
26
46
  await store.fileIndex?.record(chunks.filter((chunk) => chunk.attachment?.url));
@@ -32,7 +52,24 @@ export function findChunksOfType(snap, { chatId, type, ...bounds }) {
32
52
  .find(CHUNKS_BY_TYPE, keyRange(prefix, 'seq', bounds))
33
53
  .toArray();
34
54
  }
35
- export async function listChunks(store, { chatId, type, types, ...bounds }) {
55
+ export async function findChunksOfTypes(db, { chatId, types, ...bounds }) {
56
+ const ordered = await readChatOrdered(db, chatId, { types, ...bounds });
57
+ if (ordered) {
58
+ return ordered;
59
+ }
60
+ const byType = await Promise.all(types.map((type) => findChunksOfType(db, { chatId, type, ...bounds })));
61
+ return byType.flat().sort((left, right) => left.seq - right.seq);
62
+ }
63
+ export async function listChunks(store, input) {
64
+ const snap = store.view.snapshot();
65
+ try {
66
+ return linkChunks(store, await readChunks(snap, input));
67
+ }
68
+ finally {
69
+ await snap.close();
70
+ }
71
+ }
72
+ export async function readChunks(db, { chatId, type, types, ...bounds }) {
36
73
  const selected = selectedChunkTypes(type, types);
37
74
  if (selected?.length === 0) {
38
75
  return [];
@@ -40,23 +77,66 @@ export async function listChunks(store, { chatId, type, types, ...bounds }) {
40
77
  if (selected && selected.length > 1 && !isHistorySelection(selected)) {
41
78
  throw new CoreError('CHUNK_TYPES_UNMAPPED');
42
79
  }
43
- const snap = store.view.snapshot();
44
- try {
45
- if (selected?.length === 1) {
46
- const prefix = { chatId, type: chunkTypeOrdinal(selected[0]) };
47
- const chunks = await snap
48
- .find(CHUNKS_BY_TYPE, keyRange(prefix, 'seq', bounds))
49
- .toArray();
50
- return linkChunks(store, chunks);
51
- }
52
- const chunks = await snap
53
- .find(selected ? HISTORY_CHUNKS : CHUNKS_BY_CHAT, keyRange({ chatId }, 'seq', bounds))
54
- .toArray();
55
- return linkChunks(store, chunks);
80
+ if (selected?.length === 1) {
81
+ return findChunksOfType(db, { chatId, type: selected[0], ...bounds });
56
82
  }
57
- finally {
58
- await snap.close();
83
+ const history = selected !== null;
84
+ const ordered = history && bounds.limit
85
+ ? null
86
+ : await readChatOrdered(db, chatId, {
87
+ types: history ? HISTORY_CHUNK_TYPES : null,
88
+ ...bounds
89
+ });
90
+ if (ordered) {
91
+ return ordered;
92
+ }
93
+ return db
94
+ .find(history ? HISTORY_CHUNKS : CHUNKS_BY_CHAT, keyRange({ chatId }, 'seq', bounds))
95
+ .toArray();
96
+ }
97
+ async function readChatOrdered(db, chatId, { types, ...bounds }) {
98
+ const counter = await db.get(CHUNK_COUNTERS, { chatId });
99
+ if (!counter) {
100
+ return null;
101
+ }
102
+ const chunks = await db
103
+ .find(CHAT_CHUNKS, keyRange({ chatId }, 'seq', bounds))
104
+ .toArray();
105
+ const { first, last } = seqSpan(counter.count, bounds);
106
+ if (last < first) {
107
+ return chunks.length === 0 ? chunks : null;
59
108
  }
109
+ const [low, high] = bounds.reverse ? [chunks.at(-1), chunks[0]] : [chunks[0], chunks.at(-1)];
110
+ const whole = chunks.length === last - first + 1 && low?.seq === first && high?.seq === last;
111
+ if (!whole) {
112
+ return null;
113
+ }
114
+ const settled = new Set(chunks
115
+ .filter((chunk) => TERMINAL_RUN_STATES.has(chunk.runStatus?.state ?? ''))
116
+ .map((c) => c.runId));
117
+ const selected = types ? chunks.filter((chunk) => types.includes(chunk.type)) : chunks;
118
+ return Promise.all(selected.map((chunk) => (changeable(chunk, settled) ? current(db, chunk) : chunk)));
119
+ }
120
+ function changeable(chunk, settled) {
121
+ return ((chunk.type === 'run-status' && !settled.has(chunk.runId)) ||
122
+ (chunk.type === 'approval-request' && chunk.approvalRequest?.status === 'pending') ||
123
+ (chunk.type === 'attachment' && !chunk.attachment?.deletedAt));
124
+ }
125
+ async function current(db, chunk) {
126
+ return (await db.get(CHUNKS, { id: chunk.id })) ?? chunk;
127
+ }
128
+ function seqSpan(count, { gt, gte, lt, lte, reverse, limit }) {
129
+ let first = Math.max(0, gte ?? 0, gt === undefined ? 0 : gt + 1);
130
+ let last = Math.min(count - 1, lte ?? Infinity, lt === undefined ? Infinity : lt - 1);
131
+ if (limit) {
132
+ if (reverse) {
133
+ first = Math.max(first, last - limit + 1);
134
+ }
135
+ else {
136
+ last = Math.min(last, first + limit - 1);
137
+ }
138
+ }
139
+ return { first, last };
60
140
  }
61
141
  export async function listAttachmentsByKind(store, { kind, origin, ...bounds }) {
62
142
  const mediaKind = mediaKindOrdinal(kind);
@@ -87,6 +167,15 @@ function selectedChunkTypes(type, types) {
87
167
  }
88
168
  return [...selected];
89
169
  }
170
+ export function selectsChunk({ chatId, type, types, gt, gte, lt, lte }, chunk) {
171
+ const selected = selectedChunkTypes(type, types);
172
+ return (chunk.chatId === chatId &&
173
+ (selected === null || selected.includes(chunk.type)) &&
174
+ (gt === undefined || chunk.seq > gt) &&
175
+ (gte === undefined || chunk.seq >= gte) &&
176
+ (lt === undefined || chunk.seq < lt) &&
177
+ (lte === undefined || chunk.seq <= lte));
178
+ }
90
179
  function isHistorySelection(types) {
91
180
  return (types.length === HISTORY_CHUNK_TYPES.length &&
92
181
  types.every((type) => HISTORY_CHUNK_TYPES.includes(type)));
@@ -158,6 +247,7 @@ export async function appendChunk(store, { id = randomId(), chatId, runId, type,
158
247
  capabilityArgs,
159
248
  enhance,
160
249
  sourceId,
250
+ falsyResult: falsyResultOf(payload),
161
251
  createdAt: Date.now(),
162
252
  senderMemberId: type === 'user-message' && store.ourDeviceId && b4a.equals(store.ourDeviceId, originDeviceId)
163
253
  ? store.ourMemberId || undefined
@@ -1,4 +1,5 @@
1
1
  export declare const CHUNKS = "@qvac/chunks";
2
+ export declare const CHAT_CHUNKS = "@qvac/chat-chunks";
2
3
  export declare const CHUNK_COUNTERS = "@qvac/chunk-counters";
3
4
  export declare const AGENTS = "@qvac/agents";
4
5
  export declare const APPROVAL_RULES = "@qvac/approval-rules";
@@ -1,4 +1,5 @@
1
1
  export const CHUNKS = '@qvac/chunks';
2
+ export const CHAT_CHUNKS = '@qvac/chat-chunks';
2
3
  export const CHUNK_COUNTERS = '@qvac/chunk-counters';
3
4
  export const AGENTS = '@qvac/agents';
4
5
  export const APPROVAL_RULES = '@qvac/approval-rules';
@@ -0,0 +1,6 @@
1
+ export interface StoredConfigValue {
2
+ readonly value?: unknown;
3
+ readonly falsyValue?: string | null;
4
+ }
5
+ export declare function configValue(value: unknown): StoredConfigValue;
6
+ export declare function readConfigValue({ value, falsyValue }: StoredConfigValue): unknown;
@@ -0,0 +1,14 @@
1
+ const FALSY_VALUES = new Map([
2
+ ['false', false],
3
+ ['0', 0],
4
+ ['""', '']
5
+ ]);
6
+ export function configValue(value) {
7
+ return value === false || value === 0 || value === ''
8
+ ? { value: null, falsyValue: JSON.stringify(value) }
9
+ : { value };
10
+ }
11
+ export function readConfigValue({ value, falsyValue }) {
12
+ const falsy = falsyValue ?? '';
13
+ return FALSY_VALUES.has(falsy) ? FALSY_VALUES.get(falsy) : (value ?? null);
14
+ }
@@ -2,3 +2,4 @@ import type HyperDB from 'hyperdb';
2
2
  import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
3
3
  export declare function isPurged(tx: HyperDB, chatId: string): Promise<boolean>;
4
4
  export declare function registerChatRoutes(router: Router): void;
5
+ export declare function deleteChatChunks(tx: HyperDB, chatId: string): Promise<void>;
@@ -1,5 +1,5 @@
1
1
  import { trackActivity } from './activity.js';
2
- import { CHATS, CHUNKS, CHUNKS_BY_CHAT, CHUNK_COUNTERS, INDEXING_STATUS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS, TOOL_ACTIVITY, TOOL_ACTIVITY_BY_AGENT } from '../collections.js';
2
+ import { CHATS, CHAT_CHUNKS, CHUNKS, CHUNKS_BY_CHAT, CHUNK_COUNTERS, INDEXING_STATUS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS, TOOL_ACTIVITY, TOOL_ACTIVITY_BY_AGENT } from '../collections.js';
3
3
  export async function isPurged(tx, chatId) {
4
4
  return (await tx.get(PURGED_CHATS, { chatId })) !== null;
5
5
  }
@@ -45,12 +45,19 @@ export function registerChatRoutes(router) {
45
45
  return purged;
46
46
  }));
47
47
  }
48
+ export async function deleteChatChunks(tx, chatId) {
49
+ const inChat = { gte: { chatId }, lte: { chatId } };
50
+ for (const chunk of await tx.find(CHAT_CHUNKS, inChat).toArray()) {
51
+ await tx.delete(CHAT_CHUNKS, { chatId, seq: chunk.seq });
52
+ }
53
+ }
48
54
  async function purgeChat(tx, chat) {
49
55
  const chatId = chat.id;
50
56
  const inChat = { gte: { chatId }, lte: { chatId } };
51
57
  for (const id of await idsOf(tx.find(CHUNKS_BY_CHAT, inChat))) {
52
58
  await tx.delete(CHUNKS, { id });
53
59
  }
60
+ await deleteChatChunks(tx, chatId);
54
61
  await tx.delete(CHUNK_COUNTERS, { chatId });
55
62
  for (const knowledge of await tx.find(KNOWLEDGE_BY_CHAT, inChat).toArray()) {
56
63
  const ofKnowledge = { gte: { knowledgeId: knowledge.id }, lte: { knowledgeId: knowledge.id } };
@@ -1,11 +1,15 @@
1
1
  import { devicesByWriterKey, trackActivity } from './activity.js';
2
- import { isPurged } from './chat.js';
2
+ import { deleteChatChunks, isPurged } from './chat.js';
3
3
  import { CHUNK_PAYLOAD_FIELDS, ragSourcesWithinBounds } from '../../../schema/qvac.js';
4
4
  import { chunkTypeOrdinal } from '../../../schema/indexes.js';
5
5
  import b4a from 'b4a';
6
- import { CHUNKS, CHUNKS_BY_TYPE, CHUNK_COUNTERS, DEVICES } from '../collections.js';
6
+ import { CHAT_CHUNKS, CHUNKS, CHUNKS_BY_CHAT, CHUNKS_BY_TYPE, CHUNK_COUNTERS, DEVICES } from '../collections.js';
7
7
  const TERMINAL_RUN_STATES = new Set(['completed', 'failed', 'canceled']);
8
8
  const CREDITED_CHUNK_TYPES = new Set(['run-request', 'run-status']);
9
+ async function putChunk(tx, chunk) {
10
+ await tx.insert(CHUNKS, chunk);
11
+ await tx.insert(CHAT_CHUNKS, chunk);
12
+ }
9
13
  async function runHasTerminalStatus(tx, chatId, runId) {
10
14
  const type = chunkTypeOrdinal('run-status');
11
15
  const statuses = tx.find(CHUNKS_BY_TYPE, {
@@ -99,7 +103,7 @@ export function registerChunkRoutes(router) {
99
103
  count: prevCount + 1,
100
104
  lastActivityAt: Math.max(counter?.lastActivityAt ?? 0, op.createdAt)
101
105
  });
102
- await tx.insert(CHUNKS, appended);
106
+ await putChunk(tx, appended);
103
107
  return CREDITED_CHUNK_TYPES.has(op.type) ? appended : undefined;
104
108
  }));
105
109
  router.add('@qvac/update-chunk', trackActivity(async (op, { tx, node, log }) => {
@@ -122,7 +126,7 @@ export function registerChunkRoutes(router) {
122
126
  return;
123
127
  }
124
128
  const next = { ...chunk, runStatus: op.runStatus, updatedAt: op.updatedAt };
125
- await tx.insert(CHUNKS, next);
129
+ await putChunk(tx, next);
126
130
  return next;
127
131
  }
128
132
  if (chunk.type === 'attachment' && op.attachment !== null && op.attachment !== undefined) {
@@ -131,7 +135,7 @@ export function registerChunkRoutes(router) {
131
135
  return;
132
136
  }
133
137
  const next = { ...chunk, attachment: op.attachment, updatedAt: op.updatedAt };
134
- await tx.insert(CHUNKS, next);
138
+ await putChunk(tx, next);
135
139
  return next;
136
140
  }
137
141
  log?.warn('contract', 'update-chunk dropped: only a run-status (runStatus) or attachment (attachment) edits', { id: op.id });
@@ -161,8 +165,18 @@ export function registerChunkRoutes(router) {
161
165
  },
162
166
  updatedAt: op.resolvedAt
163
167
  };
164
- await tx.insert(CHUNKS, resolved);
168
+ await putChunk(tx, resolved);
165
169
  log?.info('contract', `approval ${op.approved ? 'approved' : 'denied'}`, { id: op.id });
166
170
  return resolved;
167
171
  }));
172
+ router.add('@qvac/reindex-chat-chunks', async ({ chatId }, { tx }) => {
173
+ if (await isPurged(tx, chatId)) {
174
+ await deleteChatChunks(tx, chatId);
175
+ return;
176
+ }
177
+ const inChat = { gte: { chatId }, lte: { chatId } };
178
+ for (const chunk of await tx.find(CHUNKS_BY_CHAT, inChat).toArray()) {
179
+ await tx.insert(CHAT_CHUNKS, chunk);
180
+ }
181
+ });
168
182
  }
@@ -1,5 +1,5 @@
1
1
  import type { Router } from '../../../spec/mesh/hyperdispatch/index.js';
2
- declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh', '@qvac/purge-chat', '@qvac/name-mesh', '@qvac/put-mesh-seed'];
2
+ declare const DROPPED: readonly ['@qvac/append-chunk', '@qvac/update-chunk', '@qvac/resolve-approval', '@qvac/put-agent', '@qvac/put-approval-rule', '@qvac/seed-defaults', '@qvac/add-knowledge', '@qvac/remove-knowledge', '@qvac/add-knowledge-artifact', '@qvac/claim-indexing', '@qvac/resolve-indexing', '@qvac/put-chat', '@qvac/put-tool-configs', '@qvac/append-tool-activity', '@qvac/migrate-agent-catalog', '@qvac/reindex-agents', '@qvac/put-chat-group', '@qvac/put-joined-mesh', '@qvac/purge-chat', '@qvac/name-mesh', '@qvac/put-mesh-seed', '@qvac/reindex-chat-chunks'];
3
3
  export type DroppedComputeRoute = (typeof DROPPED)[number];
4
4
  export declare function droppedComputeRoutes(): ReadonlyArray<DroppedComputeRoute>;
5
5
  export declare function assertComputeRoute(compute: boolean, name: string): void;
@@ -20,7 +20,8 @@ const DROPPED = [
20
20
  '@qvac/put-joined-mesh',
21
21
  '@qvac/purge-chat',
22
22
  '@qvac/name-mesh',
23
- '@qvac/put-mesh-seed'
23
+ '@qvac/put-mesh-seed',
24
+ '@qvac/reindex-chat-chunks'
24
25
  ];
25
26
  const DROPPED_ROUTES = new Set(DROPPED);
26
27
  export function droppedComputeRoutes() {
@@ -82,7 +82,7 @@ export function registerDeviceRoutes(router) {
82
82
  await tx.insert(DEVICES, next);
83
83
  return next;
84
84
  }));
85
- router.add('@qvac/report-device-capabilities', trackActivity(async ({ deviceId, installedToolNames, installedSkillNames, installedModelNames, cachedModelNames, readySkillNames, toolCatalog }, { tx, node, log }) => {
85
+ router.add('@qvac/report-device-capabilities', trackActivity(async ({ deviceId, installedToolNames, installedSkillNames, installedModelNames, cachedModelNames, readySkillNames, toolCatalog, customModels }, { tx, node, log }) => {
86
86
  const existing = await tx.get(DEVICES, { id: deviceId });
87
87
  if (!existing) {
88
88
  log?.debug('contract', 'report-device-capabilities dropped: unknown device');
@@ -101,7 +101,8 @@ export function registerDeviceRoutes(router) {
101
101
  installedModelNames: installedModelNames ?? [],
102
102
  cachedModelNames: cachedModelNames ?? [],
103
103
  readySkillNames: readySkillNames ?? existing.readySkillNames,
104
- toolCatalog: toolCatalog ?? existing.toolCatalog
104
+ toolCatalog: toolCatalog ?? existing.toolCatalog,
105
+ customModels: customModels ?? existing.customModels
105
106
  };
106
107
  await tx.insert(DEVICES, next);
107
108
  return next;
@@ -47,6 +47,7 @@ export class PendingDepartures {
47
47
  seed: row.seed,
48
48
  key: row.creator ? null : row.key,
49
49
  writerSeed: row.writerSeed ?? undefined,
50
+ compute: row.compute,
50
51
  deviceId: this._deviceId() ?? undefined,
51
52
  logger: this._log,
52
53
  blindPeerKeys: this._blindPeerKeys,
@@ -1,4 +1,4 @@
1
- import type { QvacToolReportEntry } from '../../spec/mesh/hyperschema/types.d.ts';
1
+ import type { QvacCustomModelReportEntry, QvacToolReportEntry } from '../../spec/mesh/hyperschema/types.d.ts';
2
2
  import type { AppBuildMetadata } from './app-build.ts';
3
3
  import type { Mesh } from './mesh.ts';
4
4
  export interface SaveDeviceCapabilitiesInput {
@@ -9,6 +9,7 @@ export interface SaveDeviceCapabilitiesInput {
9
9
  cachedModelNames: string[];
10
10
  readySkillNames?: string[] | null;
11
11
  toolCatalog?: QvacToolReportEntry[] | null;
12
+ customModels?: QvacCustomModelReportEntry[] | null;
12
13
  }
13
14
  export declare function saveDeviceCapabilities(store: Mesh, input: SaveDeviceCapabilitiesInput, isAccount: (mesh: Mesh) => boolean): Promise<void>;
14
15
  export declare function retractInventory(store: Mesh, deviceId: Buffer, isAccount: (mesh: Mesh) => boolean): Promise<void>;
@@ -32,7 +32,8 @@ export async function retractInventory(store, deviceId, isAccount) {
32
32
  row.installedSkillNames,
33
33
  row.installedModelNames,
34
34
  row.cachedModelNames,
35
- row.toolCatalog
35
+ row.toolCatalog,
36
+ row.customModels
36
37
  ].some((list) => list?.length) ||
37
38
  (row.readySkillNames?.length ?? 0) > (trimmed.readySkillNames?.length ?? 0);
38
39
  if (leaked) {
@@ -48,7 +49,8 @@ async function hostedOnly(store, input) {
48
49
  installedModelNames: [],
49
50
  cachedModelNames: [],
50
51
  readySkillNames: readinessForAgents(input.readySkillNames ?? [], hosted),
51
- toolCatalog: []
52
+ toolCatalog: [],
53
+ customModels: []
52
54
  };
53
55
  }
54
56
  export async function advertiseDeviceVersion(store, deviceId, app) {
@@ -1,6 +1,6 @@
1
1
  import { CoreError } from '../../errors.js';
2
2
  import { linkAttachment } from '../attachment-link.js';
3
- import { getChunk, updateChunk } from '../chunk.js';
3
+ import { getChunk, readChunk, updateChunk } from '../chunk.js';
4
4
  import { uploadFile } from './upload.js';
5
5
  import b4a from 'b4a';
6
6
  export const MAX_ATTACHMENT_BYTES = 100 * 1024 * 1024;
@@ -39,7 +39,7 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
39
39
  }
40
40
  const attachment = chunk.attachment;
41
41
  if (attachment.deletedAt) {
42
- return chunk;
42
+ return readChunk(chunk);
43
43
  }
44
44
  const updated = await updateChunk(mesh, {
45
45
  id: chunkId,
@@ -48,7 +48,7 @@ export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceI
48
48
  if (!updated.attachment?.deletedAt) {
49
49
  throw new CoreError('ATTACHMENT_DELETE_DROPPED');
50
50
  }
51
- return updated;
51
+ return readChunk(updated);
52
52
  }
53
53
  };
54
54
  }
@@ -11,7 +11,7 @@ import { type IndexingProgress, type IndexingEmbedder, type RagExtractors, type
11
11
  import type { Mesh } from '../mesh.ts';
12
12
  import type { ChatFileKey } from './records/source.ts';
13
13
  import type { SettleOptions } from './processing/settle.ts';
14
- export { fullTextBudgetChars, fullTextHits, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, type VectorHit, type RagDocumentRef } from './rag/index.ts';
14
+ export { fullTextBudgetChars, fullTextHits, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, type VectorHit, type RagDocumentRef } from './rag/index.ts';
15
15
  export { isKnowledgeRoutedAttachment } from './records/index.ts';
16
16
  export { assertRemovableAt } from './records/index.ts';
17
17
  export { assertArtifactOutput } from './records/index.ts';
@@ -15,7 +15,7 @@ import { PreviewProcessor, PREVIEW_VERSION, previewId } from './preview/index.js
15
15
  import { CoreError } from '../../errors.js';
16
16
  import { errorMessage } from '../../guards.js';
17
17
  import { ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
18
- export { fullTextBudgetChars, fullTextHits, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
18
+ export { fullTextBudgetChars, fullTextHits, retrievalBudgetTokens, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
19
19
  export { isKnowledgeRoutedAttachment } from './records/index.js';
20
20
  export { assertRemovableAt } from './records/index.js';
21
21
  export { assertArtifactOutput } from './records/index.js';
@@ -332,7 +332,8 @@ export class Files extends EventEmitter {
332
332
  processing: this._processing,
333
333
  chatIds: async () => {
334
334
  const rows = await Promise.all(ids.map((id) => getKnowledge(this._mesh, id)));
335
- return [...new Set(rows.flatMap((row) => (row ? [row.chatId] : [])))];
335
+ const chatIds = rows.flatMap((row) => (row ? [row.chatId] : []));
336
+ return [...new Set(options.chatId ? [options.chatId, ...chatIds] : chatIds)];
336
337
  }
337
338
  });
338
339
  }
@@ -61,6 +61,7 @@ export declare class FileProcessing {
61
61
  enqueue(ids: Iterable<string>, options?: EnqueueOptions): Promise<void>;
62
62
  private _enqueue;
63
63
  get active(): boolean;
64
+ canProduce(id: string): Promise<boolean>;
64
65
  pending(id: string): boolean;
65
66
  prioritize(ids: string[]): void;
66
67
  private _shutdown;
@@ -118,6 +118,14 @@ export class FileProcessing {
118
118
  get active() {
119
119
  return !this._paused && !this._closed && !this._mesh.closing;
120
120
  }
121
+ async canProduce(id) {
122
+ const record = await fileRecord(this._mesh.view, id, this._processor.input);
123
+ return Boolean(record &&
124
+ !record.deletedAt &&
125
+ this._mesh.writable &&
126
+ this._processor.applies(record) &&
127
+ (await this._processor.canProduce(record, { mesh: this._mesh, deviceId: this._deviceId })));
128
+ }
121
129
  pending(id) {
122
130
  return (!this._paused &&
123
131
  !this._closed &&
@@ -3,6 +3,7 @@ import type { Files } from '../index.ts';
3
3
  import type { FileProcessing } from './index.ts';
4
4
  export interface SettleOptions {
5
5
  readonly timeoutMs?: number;
6
+ readonly chatId?: string;
6
7
  readonly signal?: AbortSignal;
7
8
  }
8
9
  export interface SettleFilesOptions extends SettleOptions {
@@ -48,22 +48,29 @@ export function settleFiles(files, options) {
48
48
  if (!processing.active) {
49
49
  return finish();
50
50
  }
51
- const waiting = snapshots
52
- .flatMap((snapshot) => snapshot.files)
53
- .some((file) => {
51
+ let waiting = false;
52
+ for (const file of snapshots.flatMap((snapshot) => snapshot.files)) {
53
+ if (waiting)
54
+ break;
54
55
  if (!wanted.has(file.fileId) || file.readyForChat)
55
- return false;
56
+ continue;
56
57
  const rag = file.processors.find((entry) => entry.processorId === 'rag');
57
- if (!rag || rag.state === 'not-applicable') {
58
- return false;
58
+ const settled = !rag ||
59
+ rag.state === 'not-applicable' ||
60
+ (rag.state === 'failed' && (!rag.retryAt || rag.retryAt > deadline));
61
+ if (settled) {
62
+ continue;
59
63
  }
60
- if (rag.state === 'failed' && (!rag.retryAt || rag.retryAt > deadline)) {
61
- return false;
62
- }
63
- return (processing.pending(file.fileId) ||
64
- rag.state === 'running' ||
65
- (rag.retryAt !== undefined && rag.retryAt <= deadline));
66
- });
64
+ waiting =
65
+ processing.pending(file.fileId) ||
66
+ rag.state === 'running' ||
67
+ (rag.retryAt !== undefined && rag.retryAt <= deadline) ||
68
+ (rag.reason === 'waiting for agent' &&
69
+ (!(await files.record(file.fileId)) || (await processing.canProduce(file.fileId))));
70
+ }
71
+ if (done) {
72
+ return;
73
+ }
67
74
  if (!waiting) {
68
75
  finish();
69
76
  }
@@ -31,6 +31,6 @@ export declare class RagProcessor implements FileProcessor {
31
31
  export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
32
32
  export { canIndexChat } from './policy.ts';
33
33
  export type { IndexingEmbedder, IndexingProgress } from './produce.ts';
34
- export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit } from './retrieval/full-text.ts';
34
+ export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.ts';
35
35
  export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
36
36
  export type { RagArtifactPayload, RagDocumentRef, RagExtractors, RagSource, VectorHit, VectorIndex } from 'rag-extract';
@@ -70,5 +70,5 @@ export class RagProcessor {
70
70
  }
71
71
  export { MAX_KNOWLEDGE_BYTES } from './limits.js';
72
72
  export { canIndexChat } from './policy.js';
73
- export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit } from './retrieval/full-text.js';
73
+ export { formatFullDocumentContext, fullTextBudgetChars, fullTextHits, isFullTextHit, isTruncatedFullTextHit, retrievalBudgetTokens } from './retrieval/full-text.js';
74
74
  export { MemoryVectorIndex, RAG_DOCUMENT_KIND, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, classifyRagDocument, createRagSourceTagFilter, encodeRagArtifact, formatRagContext, isSupportedRagDocument, isWholeDocumentQuery, referencedDocuments, referencesPendingFile, retrieveRagKnowledge, searchRagChunks, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from 'rag-extract';
@@ -2,6 +2,7 @@ import type { QvacKnowledge } from '../../../../../spec/mesh/hyperschema/types.d
2
2
  import type { Mesh } from '../../../mesh.ts';
3
3
  import { type VectorHit } from 'rag-extract';
4
4
  export declare function fullTextBudgetChars(ctxSize: number): number;
5
+ export declare function retrievalBudgetTokens(ctxSize: number): number | undefined;
5
6
  export interface IdentifiedHit {
6
7
  readonly id: string;
7
8
  }
@@ -1,5 +1,5 @@
1
1
  import * as z from 'zod';
2
- import { classifyRagDocument, extractPdfHead, extractRagDocumentFromBytes, parseRagMetadata, RAG_DOCUMENT_KIND } from 'rag-extract';
2
+ import { classifyRagDocument, DEFAULT_CHUNK_BUDGET_TOKENS, extractPdfHead, extractRagDocumentFromBytes, parseRagMetadata, RAG_DOCUMENT_KIND } from 'rag-extract';
3
3
  const DEFAULT_FULL_TEXT_BUDGET_TOKENS = 4096;
4
4
  const CHARS_PER_TOKEN = 4;
5
5
  const FULL_TEXT_KINDS = new Set([
@@ -14,6 +14,9 @@ export function fullTextBudgetChars(ctxSize) {
14
14
  const tokens = ctxSize > 0 ? Math.floor(ctxSize / 2) : DEFAULT_FULL_TEXT_BUDGET_TOKENS;
15
15
  return tokens * CHARS_PER_TOKEN;
16
16
  }
17
+ export function retrievalBudgetTokens(ctxSize) {
18
+ return ctxSize > 0 ? Math.min(DEFAULT_CHUNK_BUDGET_TOKENS, Math.floor(ctxSize / 4)) : undefined;
19
+ }
17
20
  const FULL_TEXT_HIT_PREFIX = 'full-text-';
18
21
  export function isFullTextHit(hit) {
19
22
  return hit.id.startsWith(FULL_TEXT_HIT_PREFIX);
@@ -24,6 +24,8 @@ export declare class HostedAgents {
24
24
  private readonly _log;
25
25
  private readonly _heldKey;
26
26
  private readonly _lock;
27
+ private readonly _retry;
28
+ private readonly _deadline;
27
29
  private _closed;
28
30
  private readonly _onDirty;
29
31
  constructor(opts: HostedAgentsOptions);
@@ -32,6 +34,7 @@ export declare class HostedAgents {
32
34
  close(): Promise<void>;
33
35
  private _run;
34
36
  private _pass;
37
+ private _scheduleRetry;
35
38
  private _handOver;
36
39
  private _stored;
37
40
  private _held;