@tenex-chat/backend 0.9.5 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/daemon-wrapper.cjs +34 -0
- package/dist/index.js +59268 -0
- package/dist/wrapper.js +171 -0
- package/package.json +19 -27
- package/src/agents/AgentRegistry.ts +9 -7
- package/src/agents/AgentStorage.ts +24 -1
- package/src/agents/agent-installer.ts +6 -0
- package/src/agents/agent-loader.ts +7 -2
- package/src/agents/constants.ts +10 -2
- package/src/agents/execution/AgentExecutor.ts +35 -6
- package/src/agents/execution/StreamCallbacks.ts +53 -13
- package/src/agents/execution/StreamExecutionHandler.ts +110 -16
- package/src/agents/execution/StreamSetup.ts +19 -9
- package/src/agents/execution/ToolEventHandlers.ts +112 -0
- package/src/agents/role-categories.ts +53 -0
- package/src/agents/types/runtime.ts +7 -0
- package/src/agents/types/storage.ts +7 -0
- package/src/commands/agent/import/openclaw-distiller.ts +63 -7
- package/src/commands/agent/import/openclaw-reader.ts +54 -0
- package/src/commands/agent/import/openclaw.ts +120 -29
- package/src/commands/agent/index.ts +83 -2
- package/src/commands/setup/display.ts +123 -0
- package/src/commands/setup/embed.ts +13 -13
- package/src/commands/setup/global-system-prompt.ts +15 -17
- package/src/commands/setup/image.ts +17 -20
- package/src/commands/setup/interactive.ts +37 -20
- package/src/commands/setup/llm.ts +12 -7
- package/src/commands/setup/onboarding.ts +1580 -248
- package/src/commands/setup/providers.ts +3 -3
- package/src/conversations/ConversationStore.ts +23 -2
- package/src/conversations/MessageBuilder.ts +51 -73
- package/src/conversations/formatters/utils/conversation-transcript-formatter.ts +425 -0
- package/src/conversations/search/embeddings/ConversationEmbeddingService.ts +40 -98
- package/src/conversations/search/embeddings/ConversationIndexingJob.ts +40 -52
- package/src/conversations/services/ConversationSummarizer.ts +1 -2
- package/src/conversations/types.ts +11 -0
- package/src/daemon/Daemon.ts +78 -57
- package/src/daemon/ProjectRuntime.ts +6 -12
- package/src/daemon/SubscriptionManager.ts +13 -0
- package/src/daemon/index.ts +0 -1
- package/src/event-handler/index.ts +1 -0
- package/src/index.ts +20 -1
- package/src/llm/ChunkHandler.ts +1 -1
- package/src/llm/FinishHandler.ts +28 -4
- package/src/llm/LLMConfigEditor.ts +218 -106
- package/src/llm/index.ts +0 -4
- package/src/llm/meta/MetaModelResolver.ts +3 -18
- package/src/llm/middleware/message-sanitizer.ts +153 -0
- package/src/llm/providers/ollama-models.ts +0 -38
- package/src/llm/service.ts +50 -15
- package/src/llm/types.ts +0 -12
- package/src/llm/utils/ConfigurationManager.ts +88 -465
- package/src/llm/utils/ConfigurationTester.ts +42 -185
- package/src/llm/utils/ModelSelector.ts +156 -92
- package/src/llm/utils/ProviderConfigUI.ts +10 -141
- package/src/llm/utils/models-dev-cache.ts +102 -23
- package/src/llm/utils/provider-select-prompt.ts +284 -0
- package/src/llm/utils/provider-setup.ts +81 -34
- package/src/llm/utils/variant-list-prompt.ts +361 -0
- package/src/nostr/AgentEventDecoder.ts +1 -0
- package/src/nostr/AgentEventEncoder.ts +37 -0
- package/src/nostr/AgentProfilePublisher.ts +13 -0
- package/src/nostr/AgentPublisher.ts +26 -0
- package/src/nostr/kinds.ts +1 -0
- package/src/nostr/ndkClient.ts +4 -1
- package/src/nostr/types.ts +12 -0
- package/src/prompts/fragments/25-rag-instructions.ts +22 -21
- package/src/prompts/fragments/31-agents-md-guidance.ts +7 -21
- package/src/prompts/fragments/index.ts +2 -0
- package/src/prompts/utils/systemPromptBuilder.ts +18 -28
- package/src/services/AgentDefinitionMonitor.ts +8 -0
- package/src/services/ConfigService.ts +34 -0
- package/src/services/PubkeyService.ts +7 -1
- package/src/services/compression/CompressionService.ts +133 -74
- package/src/services/compression/compression-utils.ts +110 -19
- package/src/services/config/types.ts +0 -6
- package/src/services/dispatch/AgentDispatchService.ts +79 -0
- package/src/services/intervention/InterventionService.ts +78 -5
- package/src/services/nip46/Nip46SigningService.ts +30 -1
- package/src/services/projects/ProjectContext.ts +8 -6
- package/src/services/rag/RAGCollectionRegistry.ts +199 -0
- package/src/services/rag/RAGDatabaseService.ts +2 -7
- package/src/services/rag/RAGOperations.ts +25 -45
- package/src/services/rag/RAGService.ts +0 -31
- package/src/services/rag/RagSubscriptionService.ts +71 -122
- package/src/services/rag/rag-utils.ts +13 -0
- package/src/services/ral/RALRegistry.ts +25 -184
- package/src/services/reports/ReportEmbeddingService.ts +63 -113
- package/src/services/search/UnifiedSearchService.ts +115 -4
- package/src/services/search/index.ts +1 -0
- package/src/services/search/projectFilter.ts +20 -4
- package/src/services/search/providers/ConversationSearchProvider.ts +1 -0
- package/src/services/search/providers/GenericCollectionSearchProvider.ts +81 -0
- package/src/services/search/providers/LessonSearchProvider.ts +1 -8
- package/src/services/search/providers/ReportSearchProvider.ts +1 -0
- package/src/services/search/types.ts +24 -3
- package/src/services/trust-pubkeys/SystemPubkeyListService.ts +148 -0
- package/src/services/trust-pubkeys/TrustPubkeyService.ts +70 -9
- package/src/telemetry/setup.ts +2 -13
- package/src/tools/implementations/ask.ts +3 -3
- package/src/tools/implementations/conversation_get.ts +28 -268
- package/src/tools/implementations/fs_grep.ts +6 -6
- package/src/tools/implementations/fs_read.ts +2 -0
- package/src/tools/implementations/fs_write.ts +2 -0
- package/src/tools/implementations/learn.ts +38 -50
- package/src/tools/implementations/rag_add_documents.ts +6 -4
- package/src/tools/implementations/rag_create_collection.ts +37 -4
- package/src/tools/implementations/rag_delete_collection.ts +9 -0
- package/src/tools/implementations/{search.ts → rag_search.ts} +31 -25
- package/src/tools/registry.ts +7 -8
- package/src/tools/types.ts +11 -2
- package/src/tools/utils/transcript-args.ts +13 -0
- package/src/utils/cli-theme.ts +13 -0
- package/src/utils/logger.ts +55 -0
- package/src/utils/metadataKeys.ts +17 -0
- package/src/utils/sqlEscaping.ts +39 -0
- package/src/wrapper.ts +7 -3
- package/dist/src/index.js +0 -46790
- package/dist/tenex-backend-wrapper.cjs +0 -3
- package/src/agents/execution/constants.ts +0 -16
- package/src/agents/execution/index.ts +0 -3
- package/src/agents/index.ts +0 -4
- package/src/commands/agent.ts +0 -235
- package/src/conversations/formatters/DelegationXmlFormatter.ts +0 -64
- package/src/conversations/formatters/index.ts +0 -9
- package/src/conversations/index.ts +0 -2
- package/src/conversations/utils/content-utils.ts +0 -69
- package/src/daemon/UnixSocketTransport.ts +0 -318
- package/src/event-handler/newConversation.ts +0 -165
- package/src/events/NDKProjectStatus.ts +0 -384
- package/src/events/index.ts +0 -4
- package/src/lib/json-parser.ts +0 -30
- package/src/llm/RecordingState.ts +0 -37
- package/src/llm/StreamPublisher.ts +0 -40
- package/src/llm/middleware/flight-recorder.ts +0 -188
- package/src/llm/utils/claudeCodePromptCompiler.ts +0 -141
- package/src/nostr/constants.ts +0 -38
- package/src/prompts/core/index.ts +0 -3
- package/src/prompts/index.ts +0 -21
- package/src/services/image/index.ts +0 -12
- package/src/services/status/index.ts +0 -11
- package/src/telemetry/diagnostics.ts +0 -27
- package/src/tools/implementations/rag_query.ts +0 -107
- package/src/types/index.ts +0 -46
- package/src/utils/agentFetcher.ts +0 -107
- package/src/utils/conversation-utils.ts +0 -1
- package/src/utils/process.ts +0 -49
|
@@ -8,13 +8,13 @@
|
|
|
8
8
|
* - Embeds conversation summaries (not individual messages - too expensive)
|
|
9
9
|
* - Uses existing RAG infrastructure (LanceDB, embedding providers)
|
|
10
10
|
* - Supports hybrid search (semantic + keyword fallback)
|
|
11
|
-
* - Graceful degradation when embeddings unavailable
|
|
12
11
|
* - Project isolation: filters are applied DURING vector search (prefilter)
|
|
13
|
-
* - Upsert semantics: re-indexing updates existing documents
|
|
12
|
+
* - Upsert semantics: re-indexing updates existing documents via bulkUpsert
|
|
14
13
|
*/
|
|
15
14
|
|
|
16
15
|
import { logger } from "@/utils/logger";
|
|
17
16
|
import { RAGService, type RAGDocument, type RAGQueryResult } from "@/services/rag/RAGService";
|
|
17
|
+
import { buildProjectFilter } from "@/services/search/projectFilter";
|
|
18
18
|
import { ConversationStore } from "@/conversations/ConversationStore";
|
|
19
19
|
import { conversationRegistry } from "@/conversations/ConversationRegistry";
|
|
20
20
|
|
|
@@ -261,40 +261,17 @@ export class ConversationEmbeddingService {
|
|
|
261
261
|
): Promise<boolean> {
|
|
262
262
|
await this.ensureInitialized();
|
|
263
263
|
|
|
264
|
-
|
|
265
|
-
const result = this.buildDocument(conversationId, projectId, store);
|
|
266
|
-
|
|
267
|
-
if (result.kind !== "ok") {
|
|
268
|
-
return false;
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
const document = result.document;
|
|
272
|
-
// buildDocument always sets document.id — use it as the single source of truth
|
|
273
|
-
const documentId = document.id ?? `conv_${projectId}_${conversationId}`;
|
|
274
|
-
|
|
275
|
-
// Delete existing document before inserting (upsert semantics)
|
|
276
|
-
// This prevents duplicate entries when re-indexing
|
|
277
|
-
try {
|
|
278
|
-
await this.ragService.deleteDocumentById(CONVERSATION_COLLECTION, documentId);
|
|
279
|
-
logger.debug(`Deleted existing embedding for ${conversationId.substring(0, 8)}`);
|
|
280
|
-
} catch {
|
|
281
|
-
// Document might not exist - that's fine
|
|
282
|
-
}
|
|
264
|
+
const result = this.buildDocument(conversationId, projectId, store);
|
|
283
265
|
|
|
284
|
-
|
|
285
|
-
await this.ragService.addDocuments(CONVERSATION_COLLECTION, [document]);
|
|
286
|
-
|
|
287
|
-
logger.debug(`Indexed conversation ${conversationId.substring(0, 8)} for project ${projectId}`);
|
|
288
|
-
return true;
|
|
289
|
-
} catch (error) {
|
|
290
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
291
|
-
logger.error("Failed to index conversation", {
|
|
292
|
-
conversationId: conversationId.substring(0, 8),
|
|
293
|
-
error: message,
|
|
294
|
-
});
|
|
295
|
-
// Don't throw - indexing failures shouldn't break the application
|
|
266
|
+
if (result.kind !== "ok") {
|
|
296
267
|
return false;
|
|
297
268
|
}
|
|
269
|
+
|
|
270
|
+
// Atomic upsert via mergeInsert — one LanceDB version per chunk
|
|
271
|
+
await this.ragService.bulkUpsert(CONVERSATION_COLLECTION, [result.document]);
|
|
272
|
+
|
|
273
|
+
logger.debug(`Indexed conversation ${conversationId.substring(0, 8)} for project ${projectId}`);
|
|
274
|
+
return true;
|
|
298
275
|
}
|
|
299
276
|
|
|
300
277
|
/**
|
|
@@ -349,23 +326,9 @@ export class ConversationEmbeddingService {
|
|
|
349
326
|
}
|
|
350
327
|
|
|
351
328
|
/**
|
|
352
|
-
*
|
|
353
|
-
*
|
|
354
|
-
|
|
355
|
-
private buildProjectFilter(projectId?: string): string | undefined {
|
|
356
|
-
if (!projectId || projectId.toLowerCase() === "all") {
|
|
357
|
-
return undefined;
|
|
358
|
-
}
|
|
359
|
-
// Filter on the metadata JSON string - LanceDB stores metadata as JSON string
|
|
360
|
-
// We need to match the projectId within the serialized JSON
|
|
361
|
-
const escapedProjectId = projectId.replace(/'/g, "''");
|
|
362
|
-
return `metadata LIKE '%"projectId":"${escapedProjectId}"%'`;
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
/**
|
|
366
|
-
* Perform semantic search on conversations
|
|
367
|
-
* FIX #1: Project filter is now applied DURING vector search (prefilter),
|
|
368
|
-
* ensuring proper project isolation without leakage from other projects
|
|
329
|
+
* Perform semantic search on conversations.
|
|
330
|
+
* Project filter is applied DURING vector search (prefilter),
|
|
331
|
+
* ensuring proper project isolation without leakage from other projects.
|
|
369
332
|
*/
|
|
370
333
|
public async semanticSearch(
|
|
371
334
|
query: string,
|
|
@@ -375,42 +338,30 @@ export class ConversationEmbeddingService {
|
|
|
375
338
|
|
|
376
339
|
const { limit = 20, minScore = 0.3, projectId } = options;
|
|
377
340
|
|
|
378
|
-
|
|
379
|
-
logger.info("🔍 Semantic search", { query, limit, minScore, projectId });
|
|
380
|
-
|
|
381
|
-
// FIX #1: Build SQL filter for project isolation - applied DURING vector search
|
|
382
|
-
const filter = this.buildProjectFilter(projectId);
|
|
383
|
-
|
|
384
|
-
// Perform RAG query with prefilter for project isolation
|
|
385
|
-
// Request more results to account for minScore filtering
|
|
386
|
-
const results = await this.ragService.queryWithFilter(
|
|
387
|
-
CONVERSATION_COLLECTION,
|
|
388
|
-
query,
|
|
389
|
-
limit * 2, // Request more to filter by minScore
|
|
390
|
-
filter
|
|
391
|
-
);
|
|
392
|
-
|
|
393
|
-
// Transform and filter by minScore only (project filtering already done)
|
|
394
|
-
const searchResults: SemanticSearchResult[] = results
|
|
395
|
-
.filter((result: RAGQueryResult) => result.score >= minScore)
|
|
396
|
-
.slice(0, limit)
|
|
397
|
-
.map((result: RAGQueryResult) => this.transformResult(result));
|
|
398
|
-
|
|
399
|
-
logger.info("✅ Semantic search complete", {
|
|
400
|
-
query,
|
|
401
|
-
found: searchResults.length,
|
|
402
|
-
limit,
|
|
403
|
-
projectFilter: filter || "none",
|
|
404
|
-
});
|
|
341
|
+
logger.info("🔍 Semantic search", { query, limit, minScore, projectId });
|
|
405
342
|
|
|
406
|
-
|
|
407
|
-
} catch (error) {
|
|
408
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
409
|
-
logger.error("Semantic search failed", { query, error: message });
|
|
343
|
+
const filter = buildProjectFilter(projectId);
|
|
410
344
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
345
|
+
const results = await this.ragService.queryWithFilter(
|
|
346
|
+
CONVERSATION_COLLECTION,
|
|
347
|
+
query,
|
|
348
|
+
limit * 2,
|
|
349
|
+
filter
|
|
350
|
+
);
|
|
351
|
+
|
|
352
|
+
const searchResults: SemanticSearchResult[] = results
|
|
353
|
+
.filter((result: RAGQueryResult) => result.score >= minScore)
|
|
354
|
+
.slice(0, limit)
|
|
355
|
+
.map((result: RAGQueryResult) => this.transformResult(result));
|
|
356
|
+
|
|
357
|
+
logger.info("✅ Semantic search complete", {
|
|
358
|
+
query,
|
|
359
|
+
found: searchResults.length,
|
|
360
|
+
limit,
|
|
361
|
+
projectFilter: filter || "none",
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
return searchResults;
|
|
414
365
|
}
|
|
415
366
|
|
|
416
367
|
/**
|
|
@@ -435,13 +386,9 @@ export class ConversationEmbeddingService {
|
|
|
435
386
|
* Check if the service has any indexed conversations
|
|
436
387
|
*/
|
|
437
388
|
public async hasIndexedConversations(): Promise<boolean> {
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
return collections.includes(CONVERSATION_COLLECTION);
|
|
442
|
-
} catch {
|
|
443
|
-
return false;
|
|
444
|
-
}
|
|
389
|
+
await this.ensureInitialized();
|
|
390
|
+
const collections = await this.ragService.listCollections();
|
|
391
|
+
return collections.includes(CONVERSATION_COLLECTION);
|
|
445
392
|
}
|
|
446
393
|
|
|
447
394
|
/**
|
|
@@ -456,14 +403,9 @@ export class ConversationEmbeddingService {
|
|
|
456
403
|
* Clear all conversation embeddings
|
|
457
404
|
*/
|
|
458
405
|
public async clearIndex(): Promise<void> {
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
logger.info("Cleared conversation embeddings index");
|
|
462
|
-
} catch (error) {
|
|
463
|
-
logger.debug("No index to clear or error clearing", { error });
|
|
464
|
-
}
|
|
406
|
+
await this.ragService.deleteCollection(CONVERSATION_COLLECTION);
|
|
407
|
+
logger.info("Cleared conversation embeddings index");
|
|
465
408
|
|
|
466
|
-
// Reset initialization state
|
|
467
409
|
this.initialized = false;
|
|
468
410
|
this.initializationPromise = null;
|
|
469
411
|
}
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* - Indexes conversations across all projects
|
|
12
12
|
* - Tracks indexing state durably to avoid redundant work
|
|
13
13
|
* - Re-indexes when conversation metadata changes
|
|
14
|
-
* -
|
|
14
|
+
* - Errors propagate to scheduleNextBatch which catches and reschedules
|
|
15
15
|
* - Prevents overlapping batches
|
|
16
16
|
* - Decoupled from ConversationRegistry for multi-project support
|
|
17
17
|
*/
|
|
@@ -160,58 +160,50 @@ export class ConversationIndexingJob {
|
|
|
160
160
|
const projectIds = listProjectIdsFromDisk(this.projectsBasePath);
|
|
161
161
|
|
|
162
162
|
for (const projectId of projectIds) {
|
|
163
|
-
|
|
164
|
-
|
|
163
|
+
const conversationIds = listConversationIdsFromDiskForProject(
|
|
164
|
+
this.projectsBasePath,
|
|
165
|
+
projectId
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
for (const conversationId of conversationIds) {
|
|
169
|
+
totalChecked++;
|
|
170
|
+
|
|
171
|
+
// Check if conversation needs indexing using durable state
|
|
172
|
+
const needsIndexing = this.stateManager.needsIndexing(
|
|
165
173
|
this.projectsBasePath,
|
|
174
|
+
projectId,
|
|
175
|
+
conversationId
|
|
176
|
+
);
|
|
177
|
+
|
|
178
|
+
if (!needsIndexing) {
|
|
179
|
+
totalSkipped++;
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Build document without writing
|
|
184
|
+
const result: BuildDocumentResult = conversationEmbeddingService.buildDocument(
|
|
185
|
+
conversationId,
|
|
166
186
|
projectId
|
|
167
187
|
);
|
|
168
188
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
projectId,
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
conversationId,
|
|
187
|
-
projectId
|
|
188
|
-
);
|
|
189
|
-
|
|
190
|
-
switch (result.kind) {
|
|
191
|
-
case "ok":
|
|
192
|
-
pendingDocuments.push(result.document);
|
|
193
|
-
pendingMarkIndexed.push({ projectId, conversationId });
|
|
194
|
-
break;
|
|
195
|
-
case "noContent":
|
|
196
|
-
pendingMarkNoContent.push({ projectId, conversationId });
|
|
197
|
-
break;
|
|
198
|
-
case "error":
|
|
199
|
-
// Transient error — leave unmarked so it retries next cycle
|
|
200
|
-
totalFailed++;
|
|
201
|
-
logger.warn("Transient error building document, will retry", {
|
|
202
|
-
conversationId: conversationId.substring(0, 8),
|
|
203
|
-
projectId,
|
|
204
|
-
reason: result.reason,
|
|
205
|
-
});
|
|
206
|
-
break;
|
|
207
|
-
}
|
|
189
|
+
switch (result.kind) {
|
|
190
|
+
case "ok":
|
|
191
|
+
pendingDocuments.push(result.document);
|
|
192
|
+
pendingMarkIndexed.push({ projectId, conversationId });
|
|
193
|
+
break;
|
|
194
|
+
case "noContent":
|
|
195
|
+
pendingMarkNoContent.push({ projectId, conversationId });
|
|
196
|
+
break;
|
|
197
|
+
case "error":
|
|
198
|
+
// Transient error — leave unmarked so it retries next cycle
|
|
199
|
+
totalFailed++;
|
|
200
|
+
logger.warn("Transient error building document, will retry", {
|
|
201
|
+
conversationId: conversationId.substring(0, 8),
|
|
202
|
+
projectId,
|
|
203
|
+
reason: result.reason,
|
|
204
|
+
});
|
|
205
|
+
break;
|
|
208
206
|
}
|
|
209
|
-
} catch (error) {
|
|
210
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
211
|
-
logger.error("Failed to process project conversations", {
|
|
212
|
-
projectId,
|
|
213
|
-
error: message,
|
|
214
|
-
});
|
|
215
207
|
}
|
|
216
208
|
}
|
|
217
209
|
|
|
@@ -267,10 +259,6 @@ export class ConversationIndexingJob {
|
|
|
267
259
|
} else {
|
|
268
260
|
logger.debug("No conversations to index in this batch");
|
|
269
261
|
}
|
|
270
|
-
} catch (error) {
|
|
271
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
272
|
-
logger.error("Conversation indexing batch failed", { error: message });
|
|
273
|
-
// Don't throw - we want the job to keep running even if one batch fails
|
|
274
262
|
} finally {
|
|
275
263
|
this.isBatchRunning = false;
|
|
276
264
|
}
|
|
@@ -75,7 +75,7 @@ export class ConversationSummarizer {
|
|
|
75
75
|
}
|
|
76
76
|
|
|
77
77
|
// Get existing categories for consistency
|
|
78
|
-
const existingCategories = await this.categoryManager.getCategories();
|
|
78
|
+
const existingCategories = (await this.categoryManager.getCategories()).slice(0, 10);
|
|
79
79
|
const categoryListText = existingCategories.length > 0
|
|
80
80
|
? `Existing categories (prefer these for consistency): ${existingCategories.join(", ")}`
|
|
81
81
|
: "No existing categories yet. Create new ones as needed.";
|
|
@@ -165,7 +165,6 @@ export class ConversationSummarizer {
|
|
|
165
165
|
),
|
|
166
166
|
categories: z
|
|
167
167
|
.array(z.string())
|
|
168
|
-
.max(3)
|
|
169
168
|
.describe(
|
|
170
169
|
"0-3 category tags. Lowercase singular nouns. Prefer canonical list; create new only if necessary; may be empty []."
|
|
171
170
|
),
|
|
@@ -44,6 +44,17 @@ export interface ConversationEntry {
|
|
|
44
44
|
* Used to ensure compressed summaries are rendered as "system" role, not "user".
|
|
45
45
|
*/
|
|
46
46
|
role?: "user" | "assistant" | "tool" | "system";
|
|
47
|
+
/**
|
|
48
|
+
* Human-readable summary of a tool call, generated by the tool's getHumanReadableContent().
|
|
49
|
+
* Stored at creation time so the compression system can use the tool's own formatting
|
|
50
|
+
* without needing access to the tool registry.
|
|
51
|
+
*/
|
|
52
|
+
humanReadable?: string;
|
|
53
|
+
/**
|
|
54
|
+
* XML transcript attributes captured from tool input at execution time.
|
|
55
|
+
* Example: { description: "...", file_path: "/repo/file.ts" }.
|
|
56
|
+
*/
|
|
57
|
+
transcriptToolAttributes?: Record<string, string>;
|
|
47
58
|
/**
|
|
48
59
|
* For delegation-marker messageType: contains the marker data.
|
|
49
60
|
* This allows lazy expansion of delegation transcripts when building messages.
|
package/src/daemon/Daemon.ts
CHANGED
|
@@ -27,8 +27,6 @@ import { DaemonRouter } from "./routing/DaemonRouter";
|
|
|
27
27
|
import type { DaemonStatus } from "./types";
|
|
28
28
|
import { createEventSpan, endSpanSuccess, endSpanError, addRoutingEvent } from "./utils/telemetry";
|
|
29
29
|
import { logDropped, logRouted } from "./utils/routing-log";
|
|
30
|
-
import { UnixSocketTransport } from "./UnixSocketTransport";
|
|
31
|
-
import { streamPublisher } from "@/llm";
|
|
32
30
|
import { getConversationIndexingJob } from "@/conversations/search/embeddings";
|
|
33
31
|
import { getLanceDBMaintenanceService } from "@/services/rag/LanceDBMaintenanceService";
|
|
34
32
|
import { ConversationStore } from "@/conversations/ConversationStore";
|
|
@@ -40,6 +38,7 @@ import { RALRegistry } from "@/services/ral/RALRegistry";
|
|
|
40
38
|
import { RestartState } from "./RestartState";
|
|
41
39
|
import { AgentDefinitionMonitor } from "@/services/AgentDefinitionMonitor";
|
|
42
40
|
import { APNsService } from "@/services/apns";
|
|
41
|
+
import { getTrustPubkeyService } from "@/services/trust-pubkeys";
|
|
43
42
|
const lessonTracer = trace.getTracer("tenex.lessons");
|
|
44
43
|
|
|
45
44
|
/**
|
|
@@ -62,7 +61,6 @@ export class Daemon {
|
|
|
62
61
|
private isRunning = false;
|
|
63
62
|
private shutdownHandlers: Array<() => Promise<void>> = [];
|
|
64
63
|
private lockfile: Lockfile | null = null;
|
|
65
|
-
private streamTransport: UnixSocketTransport | null = null;
|
|
66
64
|
|
|
67
65
|
// Runtime management delegated to RuntimeLifecycle
|
|
68
66
|
private runtimeLifecycle: RuntimeLifecycle | null = null;
|
|
@@ -73,6 +71,9 @@ export class Daemon {
|
|
|
73
71
|
// Agent pubkey mapping for routing (pubkey -> project IDs)
|
|
74
72
|
private agentPubkeyToProjects = new Map<Hexpubkey, Set<string>>();
|
|
75
73
|
|
|
74
|
+
// Agent pubkeys seeded from AgentStorage at startup (covers not-yet-running projects)
|
|
75
|
+
private storedAgentPubkeys = new Set<Hexpubkey>();
|
|
76
|
+
|
|
76
77
|
// Tracked agent definition IDs for lesson subscription sync
|
|
77
78
|
private trackedLessonDefinitionIds = new Set<string>();
|
|
78
79
|
|
|
@@ -178,9 +179,10 @@ export class Daemon {
|
|
|
178
179
|
await initNDK();
|
|
179
180
|
this.ndk = getNDK();
|
|
180
181
|
|
|
181
|
-
// 6.
|
|
182
|
+
// 6. Set backend signer on NDK for NIP-42 relay auth + publish profile
|
|
182
183
|
logger.debug("Publishing backend profile");
|
|
183
184
|
const backendSigner = await config.getBackendSigner();
|
|
185
|
+
this.ndk.signer = backendSigner;
|
|
184
186
|
const backendName = loadedConfig.backendName || "tenex backend";
|
|
185
187
|
await AgentProfilePublisher.publishBackendProfile(backendSigner, backendName, this.whitelistedPubkeys);
|
|
186
188
|
|
|
@@ -214,28 +216,32 @@ export class Daemon {
|
|
|
214
216
|
this.routingLogger
|
|
215
217
|
);
|
|
216
218
|
|
|
219
|
+
// 8b. Seed trust service with all known agent pubkeys from storage
|
|
220
|
+
// This covers agents from not-yet-running projects for cross-project trust
|
|
221
|
+
await agentStorage.initialize();
|
|
222
|
+
this.storedAgentPubkeys = await agentStorage.getAllKnownPubkeys();
|
|
223
|
+
if (this.storedAgentPubkeys.size > 0) {
|
|
224
|
+
getTrustPubkeyService().setGlobalAgentPubkeys(this.storedAgentPubkeys);
|
|
225
|
+
logger.info("Seeded trust service with stored agent pubkeys", {
|
|
226
|
+
count: this.storedAgentPubkeys.size,
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
|
|
217
230
|
// 9. Start subscription immediately
|
|
218
231
|
// Projects will be discovered naturally as events arrive
|
|
219
232
|
logger.debug("Starting subscription manager");
|
|
220
233
|
await this.subscriptionManager.start();
|
|
221
234
|
logger.debug("Subscription manager started");
|
|
222
235
|
|
|
223
|
-
// 10. Start
|
|
224
|
-
logger.debug("Starting local streaming socket");
|
|
225
|
-
this.streamTransport = new UnixSocketTransport();
|
|
226
|
-
await this.streamTransport.start();
|
|
227
|
-
streamPublisher.setTransport(this.streamTransport);
|
|
228
|
-
logger.info("Local streaming socket started", { path: this.streamTransport.getSocketPath() });
|
|
229
|
-
|
|
230
|
-
// 11. Start automatic conversation indexing job
|
|
236
|
+
// 10. Start automatic conversation indexing job
|
|
231
237
|
getConversationIndexingJob().start();
|
|
232
238
|
logger.info("Automatic conversation indexing job started");
|
|
233
239
|
|
|
234
|
-
//
|
|
240
|
+
// 10b. Start LanceDB maintenance service (periodic compaction)
|
|
235
241
|
getLanceDBMaintenanceService().start();
|
|
236
242
|
logger.info("LanceDB maintenance service started");
|
|
237
243
|
|
|
238
|
-
//
|
|
244
|
+
// 11. Initialize InterventionService (after projects are loaded)
|
|
239
245
|
// This must happen after subscriptions start so agent slugs can be resolved
|
|
240
246
|
logger.debug("Initializing intervention service");
|
|
241
247
|
const interventionService = InterventionService.getInstance();
|
|
@@ -250,20 +256,20 @@ export class Daemon {
|
|
|
250
256
|
|
|
251
257
|
await interventionService.initialize();
|
|
252
258
|
|
|
253
|
-
//
|
|
259
|
+
// 11b. Initialize APNs push notification service
|
|
254
260
|
logger.debug("Initializing APNs service");
|
|
255
261
|
await APNsService.getInstance().initialize();
|
|
256
262
|
|
|
257
|
-
//
|
|
263
|
+
// 12. Initialize restart state manager
|
|
258
264
|
logger.debug("Initializing restart state manager");
|
|
259
265
|
this.restartState = new RestartState(this.daemonDir);
|
|
260
266
|
|
|
261
|
-
//
|
|
267
|
+
// 13. Setup RAL completion listener for graceful restart
|
|
262
268
|
if (this.supervisedMode) {
|
|
263
269
|
this.setupRALCompletionListener();
|
|
264
270
|
}
|
|
265
271
|
|
|
266
|
-
//
|
|
272
|
+
// 14. Start agent definition monitor for auto-upgrades
|
|
267
273
|
logger.debug("Starting agent definition monitor");
|
|
268
274
|
this.agentDefinitionMonitor = new AgentDefinitionMonitor(
|
|
269
275
|
this.ndk,
|
|
@@ -273,7 +279,7 @@ export class Daemon {
|
|
|
273
279
|
await this.agentDefinitionMonitor.start();
|
|
274
280
|
logger.info("Agent definition monitor started");
|
|
275
281
|
|
|
276
|
-
//
|
|
282
|
+
// 15. Setup graceful shutdown
|
|
277
283
|
this.setupShutdownHandlers();
|
|
278
284
|
|
|
279
285
|
this.isRunning = true;
|
|
@@ -822,6 +828,9 @@ export class Daemon {
|
|
|
822
828
|
// Sync per-agent lesson subscriptions: add new, remove stale
|
|
823
829
|
this.syncLessonSubscriptions(allAgentDefinitionIds);
|
|
824
830
|
|
|
831
|
+
// Sync trust service with all known agent pubkeys (cross-project trust)
|
|
832
|
+
this.syncTrustServiceAgentPubkeys();
|
|
833
|
+
|
|
825
834
|
// Set up callback for dynamic agent additions (e.g., via agents_write tool)
|
|
826
835
|
// This ensures new agents are immediately routable without requiring a restart
|
|
827
836
|
const context = runtime.getContext();
|
|
@@ -865,6 +874,23 @@ export class Daemon {
|
|
|
865
874
|
this.trackedLessonDefinitionIds = new Set(currentDefinitionIds);
|
|
866
875
|
}
|
|
867
876
|
|
|
877
|
+
/**
|
|
878
|
+
* Push current agent pubkeys to TrustPubkeyService for cross-project trust.
|
|
879
|
+
* Unions the daemon-level runtime pubkeys (from currently running projects)
|
|
880
|
+
* with the stored pubkeys seeded from AgentStorage at startup (covers
|
|
881
|
+
* not-yet-running projects), so trust is never dropped for known agents.
|
|
882
|
+
*/
|
|
883
|
+
private syncTrustServiceAgentPubkeys(): void {
|
|
884
|
+
const allPubkeys = new Set<Hexpubkey>(this.agentPubkeyToProjects.keys());
|
|
885
|
+
|
|
886
|
+
// Union with stored pubkeys so non-running projects retain trust
|
|
887
|
+
for (const pubkey of this.storedAgentPubkeys) {
|
|
888
|
+
allPubkeys.add(pubkey);
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
getTrustPubkeyService().setGlobalAgentPubkeys(allPubkeys);
|
|
892
|
+
}
|
|
893
|
+
|
|
868
894
|
/**
|
|
869
895
|
* Handle a dynamically added agent (e.g., created via agents_write tool).
|
|
870
896
|
* Updates the routing map and subscription to make the agent immediately routable.
|
|
@@ -879,6 +905,9 @@ export class Daemon {
|
|
|
879
905
|
projectSet.add(projectId);
|
|
880
906
|
}
|
|
881
907
|
|
|
908
|
+
// Persist in stored set so trust survives if this project later stops
|
|
909
|
+
this.storedAgentPubkeys.add(agent.pubkey);
|
|
910
|
+
|
|
882
911
|
// Update subscriptions
|
|
883
912
|
if (this.subscriptionManager) {
|
|
884
913
|
const allPubkeys = Array.from(this.agentPubkeyToProjects.keys());
|
|
@@ -895,6 +924,9 @@ export class Daemon {
|
|
|
895
924
|
const dTag = projectId.split(":").slice(2).join(":");
|
|
896
925
|
OwnerAgentListService.getInstance().registerAgents(dTag, [agent.pubkey]);
|
|
897
926
|
|
|
927
|
+
// Sync trust service with updated agent pubkeys (cross-project trust)
|
|
928
|
+
this.syncTrustServiceAgentPubkeys();
|
|
929
|
+
|
|
898
930
|
logger.info("Dynamic agent added to routing", {
|
|
899
931
|
projectId,
|
|
900
932
|
agentSlug: agent.slug,
|
|
@@ -1336,6 +1368,17 @@ export class Daemon {
|
|
|
1336
1368
|
* Setup graceful shutdown handlers
|
|
1337
1369
|
*/
|
|
1338
1370
|
private setupShutdownHandlers(): void {
|
|
1371
|
+
// Prevent EPIPE from crashing the daemon when stdout/stderr pipe breaks
|
|
1372
|
+
// (e.g. when the TUI exits). The daemon logs to ~/.tenex/daemon/daemon.log
|
|
1373
|
+
// so silently dropping broken-pipe writes is correct behavior.
|
|
1374
|
+
process.stdout.on('error', (err) => {
|
|
1375
|
+
if ((err as NodeJS.ErrnoException).code === 'EPIPE') return;
|
|
1376
|
+
throw err;
|
|
1377
|
+
});
|
|
1378
|
+
process.stderr.on('error', (err) => {
|
|
1379
|
+
if ((err as NodeJS.ErrnoException).code === 'EPIPE') return;
|
|
1380
|
+
throw err;
|
|
1381
|
+
});
|
|
1339
1382
|
/**
|
|
1340
1383
|
* Perform graceful shutdown of the daemon.
|
|
1341
1384
|
* @param exitCode - Exit code to use (default: 0)
|
|
@@ -1362,88 +1405,70 @@ export class Daemon {
|
|
|
1362
1405
|
console.log(`[Daemon] Saved ${bootedProjects.length} booted project(s) for restart`);
|
|
1363
1406
|
}
|
|
1364
1407
|
|
|
1365
|
-
if (this.streamTransport) {
|
|
1366
|
-
process.stdout.write("Stopping stream transport...");
|
|
1367
|
-
await this.streamTransport.stop();
|
|
1368
|
-
streamPublisher.setTransport(null);
|
|
1369
|
-
this.streamTransport = null;
|
|
1370
|
-
console.log(" done");
|
|
1371
|
-
}
|
|
1372
|
-
|
|
1373
1408
|
// Stop conversation indexing job
|
|
1374
|
-
|
|
1409
|
+
logger.info("Stopping conversation indexing job...");
|
|
1375
1410
|
getConversationIndexingJob().stop();
|
|
1376
|
-
console.log(" done");
|
|
1377
1411
|
|
|
1378
1412
|
// Stop LanceDB maintenance service
|
|
1379
|
-
|
|
1413
|
+
logger.info("Stopping LanceDB maintenance service...");
|
|
1380
1414
|
getLanceDBMaintenanceService().stop();
|
|
1381
|
-
console.log(" done");
|
|
1382
1415
|
|
|
1383
1416
|
// Stop agent definition monitor
|
|
1384
1417
|
if (this.agentDefinitionMonitor) {
|
|
1385
|
-
|
|
1418
|
+
logger.info("Stopping agent definition monitor...");
|
|
1386
1419
|
this.agentDefinitionMonitor.stop();
|
|
1387
1420
|
this.agentDefinitionMonitor = null;
|
|
1388
|
-
console.log(" done");
|
|
1389
1421
|
}
|
|
1390
1422
|
|
|
1391
1423
|
// Stop intervention service
|
|
1392
|
-
|
|
1424
|
+
logger.info("Stopping intervention service...");
|
|
1393
1425
|
InterventionService.getInstance().shutdown();
|
|
1394
|
-
console.log(" done");
|
|
1395
1426
|
|
|
1396
1427
|
// Stop owner agent list service
|
|
1397
|
-
|
|
1428
|
+
logger.info("Stopping owner agent list service...");
|
|
1398
1429
|
OwnerAgentListService.getInstance().shutdown();
|
|
1399
|
-
console.log(" done");
|
|
1400
1430
|
|
|
1401
1431
|
// Stop NIP-46 signing service
|
|
1402
|
-
|
|
1432
|
+
logger.info("Stopping NIP-46 signing service...");
|
|
1403
1433
|
await Nip46SigningService.getInstance().shutdown();
|
|
1404
|
-
console.log(" done");
|
|
1405
1434
|
|
|
1406
1435
|
if (this.subscriptionManager) {
|
|
1407
|
-
|
|
1436
|
+
logger.info("Stopping subscriptions...");
|
|
1408
1437
|
this.subscriptionManager.stop();
|
|
1409
|
-
console.log(" done");
|
|
1410
1438
|
}
|
|
1411
1439
|
|
|
1412
1440
|
if (this.runtimeLifecycle) {
|
|
1413
1441
|
const stats = this.runtimeLifecycle.getStats();
|
|
1414
1442
|
if (stats.activeCount > 0) {
|
|
1415
|
-
|
|
1443
|
+
logger.info(`Stopping ${stats.activeCount} project runtime(s)...`);
|
|
1416
1444
|
}
|
|
1417
1445
|
await this.runtimeLifecycle.stopAllRuntimes();
|
|
1418
1446
|
}
|
|
1419
1447
|
|
|
1420
1448
|
// Close the global prefix KV store (after all runtimes are stopped)
|
|
1421
|
-
|
|
1449
|
+
logger.info("Closing storage...");
|
|
1422
1450
|
await prefixKVStore.forceClose();
|
|
1423
|
-
console.log(" done");
|
|
1424
1451
|
|
|
1425
1452
|
if (this.shutdownHandlers.length > 0) {
|
|
1426
|
-
|
|
1453
|
+
logger.info("Running shutdown handlers...");
|
|
1427
1454
|
for (const handler of this.shutdownHandlers) {
|
|
1428
1455
|
await handler();
|
|
1429
1456
|
}
|
|
1430
|
-
console.log(" done");
|
|
1431
1457
|
}
|
|
1432
1458
|
|
|
1433
1459
|
if (this.lockfile) {
|
|
1434
1460
|
await this.lockfile.release();
|
|
1435
1461
|
}
|
|
1436
1462
|
|
|
1437
|
-
|
|
1463
|
+
logger.info("Flushing telemetry...");
|
|
1438
1464
|
const conversationSpanManager = getConversationSpanManager();
|
|
1439
1465
|
conversationSpanManager.shutdown();
|
|
1440
1466
|
await shutdownTelemetry();
|
|
1441
|
-
console.log(" done");
|
|
1442
1467
|
|
|
1443
1468
|
if (isGracefulRestart) {
|
|
1444
|
-
|
|
1469
|
+
logger.info("[Daemon] Graceful restart complete - exiting with code 0");
|
|
1445
1470
|
} else {
|
|
1446
|
-
|
|
1471
|
+
logger.info("Shutdown complete.");
|
|
1447
1472
|
}
|
|
1448
1473
|
process.exit(exitCode);
|
|
1449
1474
|
} catch (error) {
|
|
@@ -1799,6 +1824,9 @@ export class Daemon {
|
|
|
1799
1824
|
|
|
1800
1825
|
this.subscriptionManager.updateAgentMentions(Array.from(allAgentPubkeys));
|
|
1801
1826
|
this.syncLessonSubscriptions(allAgentDefinitionIds);
|
|
1827
|
+
|
|
1828
|
+
// Sync trust service with remaining agent pubkeys (cross-project trust)
|
|
1829
|
+
this.syncTrustServiceAgentPubkeys();
|
|
1802
1830
|
} catch (error) {
|
|
1803
1831
|
logger.error("Failed to update subscription after runtime removed", {
|
|
1804
1832
|
projectId,
|
|
@@ -1819,13 +1847,6 @@ export class Daemon {
|
|
|
1819
1847
|
|
|
1820
1848
|
this.isRunning = false;
|
|
1821
1849
|
|
|
1822
|
-
// Stop streaming socket
|
|
1823
|
-
if (this.streamTransport) {
|
|
1824
|
-
await this.streamTransport.stop();
|
|
1825
|
-
streamPublisher.setTransport(null);
|
|
1826
|
-
this.streamTransport = null;
|
|
1827
|
-
}
|
|
1828
|
-
|
|
1829
1850
|
// Stop conversation indexing job
|
|
1830
1851
|
getConversationIndexingJob().stop();
|
|
1831
1852
|
|