@semiont/make-meaning 0.5.26 → 0.5.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.ts +113 -16
- package/dist/index.js +319 -131
- package/dist/index.js.map +1 -1
- package/dist/smelter-main.js +5 -6
- package/dist/smelter-main.js.map +1 -1
- package/dist/weaver-main.js +2 -1
- package/dist/weaver-main.js.map +1 -1
- package/package.json +13 -13
package/dist/index.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { STALL_THRESHOLD_MS, FsJobQueue } from '@semiont/jobs';
|
|
2
2
|
import { FilesystemViewStorage, resolveStorageUri, EventQuery, createEventStore } from '@semiont/event-sourcing';
|
|
3
|
-
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, getPrimaryRepresentation, decodeRepresentation, getBodySource, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, baseMediaType, isSupportedMediaType, capabilitiesOf, didToAgent, assembleAnnotation, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, extensionForMediaType, applyBodyOperations, softwareToAgent } from '@semiont/core';
|
|
3
|
+
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, getPrimaryRepresentation, decodeRepresentation, getBodySource, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, baseMediaType, isSupportedMediaType, capabilitiesOf, didToAgent, assembleAnnotation, isGenerationJobParams, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, extensionForMediaType, applyBodyOperations, softwareToAgent } from '@semiont/core';
|
|
4
4
|
import { recordGatherDegrade, withActorSpan, recordBusEmit, withSpan, SpanKind, registerJobQueueProvider, registerVectorIndexSizeProvider } from '@semiont/observability';
|
|
5
5
|
import { createInferenceClient } from '@semiont/inference';
|
|
6
6
|
import { compareByRecencyThenId, getGraphDatabase } from '@semiont/graph';
|
|
7
7
|
import { WorkingTreeStore, createAnchoredTextStore, deriveStorageUri, anchoredTextStoreOverTransport, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
8
8
|
import { getEntityTypes, DEFAULT_ENTITY_TYPES } from '@semiont/ontology';
|
|
9
|
+
import { mergeByResource } from '@semiont/vectors';
|
|
9
10
|
import { promises } from 'fs';
|
|
10
11
|
import * as path3 from 'path';
|
|
11
12
|
import { createGzip, createGunzip } from 'zlib';
|
|
@@ -9905,11 +9906,9 @@ async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logge
|
|
|
9905
9906
|
graph: graphDb,
|
|
9906
9907
|
weaveProgress,
|
|
9907
9908
|
smeltProgress,
|
|
9909
|
+
vectors: options.vectorStore,
|
|
9908
9910
|
projectionsDir: project.projectionsDir
|
|
9909
9911
|
};
|
|
9910
|
-
if (options?.vectorStore) {
|
|
9911
|
-
kb.vectors = options.vectorStore;
|
|
9912
|
-
}
|
|
9913
9912
|
return kb;
|
|
9914
9913
|
}
|
|
9915
9914
|
var PROJECTION_LAG_BACKOFF_MS = [25, 50, 100, 200];
|
|
@@ -10015,15 +10014,20 @@ var GraphContext = class {
|
|
|
10015
10014
|
const citedSeen = /* @__PURE__ */ new Set();
|
|
10016
10015
|
for (const ann of referencedBy) {
|
|
10017
10016
|
const source = getTargetSource(ann.target);
|
|
10018
|
-
if (!source || source === String(resourceId10) ||
|
|
10019
|
-
citedSeen.
|
|
10020
|
-
|
|
10021
|
-
|
|
10022
|
-
|
|
10017
|
+
if (!source || source === String(resourceId10) || !ann.id || seen.has(ann.id)) continue;
|
|
10018
|
+
if (!citedSeen.has(source)) {
|
|
10019
|
+
citedSeen.add(source);
|
|
10020
|
+
const view = await kb.views.get(resourceId(source));
|
|
10021
|
+
addResourceNode(source, view?.resource?.name ?? source, view?.resource ? getResourceEntityTypes(view.resource) : []);
|
|
10022
|
+
}
|
|
10023
|
+
nodes.push({ id: ann.id, type: "annotation", label: ann.motivation ?? "annotation", entityTypes: getEntityTypes(ann), annotation: ann });
|
|
10024
|
+
seen.add(ann.id);
|
|
10025
|
+
edges.push({ source: ann.id, target: source, type: "annotation-of" });
|
|
10026
|
+
edges.push({ source: ann.id, target: mainId, type: "cites" });
|
|
10023
10027
|
}
|
|
10024
10028
|
for (const ann of annotations) {
|
|
10025
10029
|
if (!ann.id || seen.has(ann.id)) continue;
|
|
10026
|
-
nodes.push({ id: ann.id, type: "annotation", label: ann.motivation ?? "annotation", entityTypes: getEntityTypes(ann) });
|
|
10030
|
+
nodes.push({ id: ann.id, type: "annotation", label: ann.motivation ?? "annotation", entityTypes: getEntityTypes(ann), annotation: ann });
|
|
10027
10031
|
seen.add(ann.id);
|
|
10028
10032
|
edges.push({ source: ann.id, target: mainId, type: "annotation-of" });
|
|
10029
10033
|
}
|
|
@@ -10057,7 +10061,19 @@ Format as a simple list, one suggestion per line.`;
|
|
|
10057
10061
|
}
|
|
10058
10062
|
return response.split("\n").map((line) => line.replace(/^[-*•]\s*/, "").trim()).filter((line) => line.length > 0).slice(0, 3);
|
|
10059
10063
|
}
|
|
10060
|
-
|
|
10064
|
+
|
|
10065
|
+
// src/graph-read-grace.ts
|
|
10066
|
+
async function resourceWithViewGrace(kb, rid) {
|
|
10067
|
+
const fromGraph = await kb.graph.getResource(rid).catch(() => null);
|
|
10068
|
+
if (fromGraph) return { resource: fromGraph, laggedBehindView: false };
|
|
10069
|
+
const view = await kb.views.get(rid);
|
|
10070
|
+
if (view) return { resource: view.resource, laggedBehindView: true };
|
|
10071
|
+
return { resource: null, laggedBehindView: false };
|
|
10072
|
+
}
|
|
10073
|
+
|
|
10074
|
+
// src/resource-context.ts
|
|
10075
|
+
var SEMANTIC_OVER_FETCH = 4;
|
|
10076
|
+
var ResourceContext = class _ResourceContext {
|
|
10061
10077
|
/**
|
|
10062
10078
|
* Get resource metadata from view storage
|
|
10063
10079
|
*/
|
|
@@ -10082,21 +10098,67 @@ var ResourceContext = class {
|
|
|
10082
10098
|
* barrier-stamped projection, so an unsearched listing is read-your-writes
|
|
10083
10099
|
* where the graph is only eventually consistent.
|
|
10084
10100
|
*/
|
|
10085
|
-
static async listResources(filters, kb) {
|
|
10101
|
+
static async listResources(filters, kb, semantic) {
|
|
10086
10102
|
const { search: rawSearch, archived, entityType: entityType2, offset = 0, limit = 50 } = filters ?? {};
|
|
10087
10103
|
const search = rawSearch?.trim() || void 0;
|
|
10088
10104
|
if (search) {
|
|
10089
|
-
|
|
10105
|
+
const lexical = await kb.graph.listResources({
|
|
10090
10106
|
search,
|
|
10091
10107
|
archived,
|
|
10092
10108
|
entityTypes: entityType2 ? [entityType2] : void 0,
|
|
10093
10109
|
offset,
|
|
10094
10110
|
limit
|
|
10095
10111
|
});
|
|
10112
|
+
if (lexical.total > 0 || offset > 0) return { ...lexical, matchKind: "lexical" };
|
|
10113
|
+
return _ResourceContext.semanticFallback(search, limit, kb, semantic);
|
|
10096
10114
|
}
|
|
10097
10115
|
const allViews = await kb.views.getAll();
|
|
10098
10116
|
const matches = allViews.map((view) => view.resource).filter((doc) => archived === void 0 || doc.archived === archived).filter((doc) => !entityType2 || getResourceEntityTypes(doc).includes(entityType2)).sort(compareByRecencyThenId);
|
|
10099
|
-
return { resources: matches.slice(offset, offset + limit), total: matches.length };
|
|
10117
|
+
return { resources: matches.slice(offset, offset + limit), total: matches.length, matchKind: "lexical" };
|
|
10118
|
+
}
|
|
10119
|
+
/**
|
|
10120
|
+
* Answer an empty lexical search from the vector index (SEMANTIC-FALLBACK):
|
|
10121
|
+
* embed the query once, fold chunk hits per resource, floor them, and label
|
|
10122
|
+
* the answer 'semantic' so the UI can say "no title matches, but these
|
|
10123
|
+
* documents discuss it".
|
|
10124
|
+
*
|
|
10125
|
+
* Degradation is the contract (axioms S3–S5): unconfigured vectors, an
|
|
10126
|
+
* absent provider, or ANY failure inside the fallback yields the same
|
|
10127
|
+
* empty page the caller already had, labelled 'lexical' — a broken
|
|
10128
|
+
* fallback must never turn a working empty search into an error.
|
|
10129
|
+
*
|
|
10130
|
+
* The floor is applied HERE rather than passed as `scoreThreshold`, so the
|
|
10131
|
+
* below-floor hits exist to be counted — the debug line is the evidence
|
|
10132
|
+
* decision #1's guessed 0.6 gets tuned from.
|
|
10133
|
+
*/
|
|
10134
|
+
static async semanticFallback(search, limit, kb, semantic) {
|
|
10135
|
+
const empty = { resources: [], total: 0, matchKind: "lexical" };
|
|
10136
|
+
try {
|
|
10137
|
+
const embedding = await semantic.embeddingProvider.embed(search);
|
|
10138
|
+
const hits = await kb.vectors.searchResources(embedding, { limit: limit * SEMANTIC_OVER_FETCH });
|
|
10139
|
+
const merged = mergeByResource(hits);
|
|
10140
|
+
const aboveFloor = merged.filter((h) => h.score >= semantic.semanticFloor);
|
|
10141
|
+
semantic.logger.debug("[search FALLBACK] semantic score distribution", {
|
|
10142
|
+
chunkHits: hits.length,
|
|
10143
|
+
resources: merged.length,
|
|
10144
|
+
aboveFloor: aboveFloor.length,
|
|
10145
|
+
belowFloor: merged.length - aboveFloor.length,
|
|
10146
|
+
topScore: merged[0]?.score,
|
|
10147
|
+
bottomScore: merged[merged.length - 1]?.score,
|
|
10148
|
+
floor: semantic.semanticFloor
|
|
10149
|
+
});
|
|
10150
|
+
const resources = [];
|
|
10151
|
+
for (const hit of aboveFloor.slice(0, limit)) {
|
|
10152
|
+
const { resource } = await resourceWithViewGrace(kb, hit.resourceId);
|
|
10153
|
+
if (resource) resources.push({ ...resource, content: hit.text });
|
|
10154
|
+
}
|
|
10155
|
+
return { resources, total: aboveFloor.length, matchKind: "semantic" };
|
|
10156
|
+
} catch (error) {
|
|
10157
|
+
semantic.logger.warn("[search FALLBACK] degraded to the empty lexical page", {
|
|
10158
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
10159
|
+
});
|
|
10160
|
+
return empty;
|
|
10161
|
+
}
|
|
10100
10162
|
}
|
|
10101
10163
|
/**
|
|
10102
10164
|
* Add content previews to resources (for search results)
|
|
@@ -10146,7 +10208,7 @@ var AnnotationContext = class {
|
|
|
10146
10208
|
* @returns Rich context for LLM processing
|
|
10147
10209
|
* @throws Error if annotation or resource not found
|
|
10148
10210
|
*/
|
|
10149
|
-
static async buildLLMContext(annotationId5, resourceId10, kb, options = {}, inferenceClient, logger
|
|
10211
|
+
static async buildLLMContext(annotationId5, resourceId10, kb, embeddingProvider, options = {}, inferenceClient, logger) {
|
|
10150
10212
|
const {
|
|
10151
10213
|
includeSourceContext = true,
|
|
10152
10214
|
includeTargetContext = true,
|
|
@@ -10278,7 +10340,7 @@ Summary:`;
|
|
|
10278
10340
|
}
|
|
10279
10341
|
}
|
|
10280
10342
|
let semanticContext;
|
|
10281
|
-
if (
|
|
10343
|
+
if (sourceContext?.selected) {
|
|
10282
10344
|
try {
|
|
10283
10345
|
const focalEmbedding = await embeddingProvider.embed(sourceContext.selected);
|
|
10284
10346
|
const results = await kb.vectors.searchAnnotations(focalEmbedding, {
|
|
@@ -10286,18 +10348,27 @@ Summary:`;
|
|
|
10286
10348
|
scoreThreshold: 0.5,
|
|
10287
10349
|
filter: { excludeResourceId: resourceId10 }
|
|
10288
10350
|
});
|
|
10289
|
-
|
|
10290
|
-
|
|
10291
|
-
|
|
10292
|
-
|
|
10293
|
-
|
|
10294
|
-
|
|
10295
|
-
|
|
10296
|
-
|
|
10297
|
-
|
|
10298
|
-
|
|
10299
|
-
|
|
10300
|
-
|
|
10351
|
+
const similar = [];
|
|
10352
|
+
for (const r of results) {
|
|
10353
|
+
const matchView = await kb.views.get(r.resourceId);
|
|
10354
|
+
const resourceName = matchView?.resource?.name;
|
|
10355
|
+
if (!resourceName) {
|
|
10356
|
+
logger?.debug("Semantic match dropped \u2014 no view for source resource", { resourceId: String(r.resourceId) });
|
|
10357
|
+
continue;
|
|
10358
|
+
}
|
|
10359
|
+
similar.push({
|
|
10360
|
+
text: r.text,
|
|
10361
|
+
resourceId: r.resourceId,
|
|
10362
|
+
resourceName,
|
|
10363
|
+
annotationId: r.annotationId,
|
|
10364
|
+
score: r.score,
|
|
10365
|
+
entityTypes: r.entityTypes,
|
|
10366
|
+
...r.machineRead ? { machineRead: true } : {}
|
|
10367
|
+
});
|
|
10368
|
+
}
|
|
10369
|
+
if (similar.length > 0) {
|
|
10370
|
+
semanticContext = { similar };
|
|
10371
|
+
logger?.debug("Semantic context found", { matches: similar.length });
|
|
10301
10372
|
}
|
|
10302
10373
|
} catch (error) {
|
|
10303
10374
|
logger?.warn("Semantic context search failed", { error });
|
|
@@ -10602,43 +10673,54 @@ var LLMContext = class {
|
|
|
10602
10673
|
if (options.includeContent) content.related = relatedContent;
|
|
10603
10674
|
let semanticContext;
|
|
10604
10675
|
const vectors = kb.vectors;
|
|
10605
|
-
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
|
|
10609
|
-
|
|
10610
|
-
|
|
10611
|
-
|
|
10612
|
-
|
|
10613
|
-
|
|
10614
|
-
|
|
10615
|
-
|
|
10616
|
-
|
|
10617
|
-
|
|
10618
|
-
|
|
10619
|
-
|
|
10620
|
-
|
|
10621
|
-
|
|
10622
|
-
|
|
10623
|
-
|
|
10624
|
-
|
|
10625
|
-
|
|
10626
|
-
|
|
10627
|
-
|
|
10628
|
-
});
|
|
10629
|
-
}
|
|
10676
|
+
const excludeEntityTypes = options.excludeEntityTypes ?? [];
|
|
10677
|
+
const search = () => vectors.searchByResource(resourceId10, {
|
|
10678
|
+
limit: options.maxResources,
|
|
10679
|
+
scoreThreshold: 0.5,
|
|
10680
|
+
...excludeEntityTypes.length ? { filter: { excludeEntityTypes } } : {}
|
|
10681
|
+
});
|
|
10682
|
+
let matches = await search();
|
|
10683
|
+
if (matches.length === 0) {
|
|
10684
|
+
const contentChecksum = getPrimaryRepresentation(mainDoc)?.checksum;
|
|
10685
|
+
if (contentChecksum) {
|
|
10686
|
+
try {
|
|
10687
|
+
const outcome = await kb.smeltProgress.whenSettled(resourceIdStr, contentChecksum, settleTimeoutMs);
|
|
10688
|
+
if (outcome === "indexed") {
|
|
10689
|
+
matches = await search();
|
|
10690
|
+
}
|
|
10691
|
+
} catch (error) {
|
|
10692
|
+
if (!(error instanceof SmeltProgressTimeout)) throw error;
|
|
10693
|
+
recordGatherDegrade("vectors");
|
|
10694
|
+
logger.warn("[gather DEGRADED] semanticContext absent \u2014 the vector projection did not settle within the barrier", {
|
|
10695
|
+
resourceId: resourceIdStr,
|
|
10696
|
+
contentChecksum,
|
|
10697
|
+
timeoutMs: settleTimeoutMs
|
|
10698
|
+
});
|
|
10630
10699
|
}
|
|
10631
10700
|
}
|
|
10632
|
-
|
|
10701
|
+
}
|
|
10702
|
+
if (matches.length > 0) {
|
|
10703
|
+
const similar = [];
|
|
10704
|
+
for (const m of matches) {
|
|
10705
|
+
const matchView = await kb.views.get(m.resourceId);
|
|
10706
|
+
const resourceName = matchView?.resource?.name;
|
|
10707
|
+
if (!resourceName) {
|
|
10708
|
+
logger?.debug("Semantic match dropped \u2014 no view for source resource", { resourceId: String(m.resourceId) });
|
|
10709
|
+
continue;
|
|
10710
|
+
}
|
|
10711
|
+
similar.push({
|
|
10712
|
+
text: m.text,
|
|
10713
|
+
resourceId: m.resourceId,
|
|
10714
|
+
resourceName,
|
|
10715
|
+
...m.annotationId ? { annotationId: m.annotationId } : {},
|
|
10716
|
+
score: m.score,
|
|
10717
|
+
...m.entityTypes ? { entityTypes: m.entityTypes } : {},
|
|
10718
|
+
...m.machineRead ? { machineRead: true } : {}
|
|
10719
|
+
});
|
|
10720
|
+
}
|
|
10721
|
+
if (similar.length > 0) {
|
|
10633
10722
|
semanticContext = {
|
|
10634
|
-
similar
|
|
10635
|
-
text: m.text,
|
|
10636
|
-
resourceId: m.resourceId,
|
|
10637
|
-
...m.annotationId ? { annotationId: m.annotationId } : {},
|
|
10638
|
-
score: m.score,
|
|
10639
|
-
...m.entityTypes ? { entityTypes: m.entityTypes } : {},
|
|
10640
|
-
...m.machineRead ? { machineRead: true } : {}
|
|
10641
|
-
})),
|
|
10723
|
+
similar,
|
|
10642
10724
|
...excludeEntityTypes.length ? { excludedEntityTypes: excludeEntityTypes } : {}
|
|
10643
10725
|
};
|
|
10644
10726
|
}
|
|
@@ -10719,10 +10801,10 @@ var Gatherer = class {
|
|
|
10719
10801
|
annotationId(event.annotationId),
|
|
10720
10802
|
resourceId(event.resourceId),
|
|
10721
10803
|
this.kb,
|
|
10804
|
+
this.embeddingProvider,
|
|
10722
10805
|
event.options ?? {},
|
|
10723
10806
|
this.inferenceClient,
|
|
10724
|
-
this.logger
|
|
10725
|
-
this.embeddingProvider
|
|
10807
|
+
this.logger
|
|
10726
10808
|
);
|
|
10727
10809
|
this.eventBus.get("gather:complete").next({
|
|
10728
10810
|
correlationId: event.correlationId,
|
|
@@ -10791,17 +10873,6 @@ var Gatherer = class {
|
|
|
10791
10873
|
// src/matcher.ts
|
|
10792
10874
|
var import_rxjs2 = __toESM(require_cjs());
|
|
10793
10875
|
var import_operators2 = __toESM(require_operators());
|
|
10794
|
-
|
|
10795
|
-
// src/graph-read-grace.ts
|
|
10796
|
-
async function resourceWithViewGrace(kb, rid) {
|
|
10797
|
-
const fromGraph = await kb.graph.getResource(rid).catch(() => null);
|
|
10798
|
-
if (fromGraph) return { resource: fromGraph, laggedBehindView: false };
|
|
10799
|
-
const view = await kb.views.get(rid);
|
|
10800
|
-
if (view) return { resource: view.resource, laggedBehindView: true };
|
|
10801
|
-
return { resource: null, laggedBehindView: false };
|
|
10802
|
-
}
|
|
10803
|
-
|
|
10804
|
-
// src/matcher.ts
|
|
10805
10876
|
var Matcher = class {
|
|
10806
10877
|
constructor(kb, eventBus, logger, inferenceClient, embeddingProvider) {
|
|
10807
10878
|
this.kb = kb;
|
|
@@ -11116,7 +11187,7 @@ For each candidate, output a line with the number and score, like:
|
|
|
11116
11187
|
* from the scorer.
|
|
11117
11188
|
*/
|
|
11118
11189
|
async searchVectors(searchTerm) {
|
|
11119
|
-
if (!
|
|
11190
|
+
if (!searchTerm.trim()) return [];
|
|
11120
11191
|
try {
|
|
11121
11192
|
const embedding = await this.embeddingProvider.embed(searchTerm);
|
|
11122
11193
|
const results = await this.kb.vectors.searchResources(embedding, {
|
|
@@ -11680,6 +11751,29 @@ var JOB_TYPES = [
|
|
|
11680
11751
|
"tag-annotation",
|
|
11681
11752
|
"generation"
|
|
11682
11753
|
];
|
|
11754
|
+
var inferencePairKey = (provider, model) => `${provider} ${model}`;
|
|
11755
|
+
function eachAdmittedInference(config, visit) {
|
|
11756
|
+
for (const jobType of JOB_TYPES) {
|
|
11757
|
+
try {
|
|
11758
|
+
visit(resolveWorkerInference(config, jobType), jobType);
|
|
11759
|
+
} catch {
|
|
11760
|
+
}
|
|
11761
|
+
}
|
|
11762
|
+
for (const actor of ["gatherer", "matcher"]) {
|
|
11763
|
+
try {
|
|
11764
|
+
visit(resolveActorInference(config, actor));
|
|
11765
|
+
} catch {
|
|
11766
|
+
}
|
|
11767
|
+
}
|
|
11768
|
+
}
|
|
11769
|
+
function deriveInferencePairs(config) {
|
|
11770
|
+
const pairs = /* @__PURE__ */ new Map();
|
|
11771
|
+
eachAdmittedInference(config, (inference) => {
|
|
11772
|
+
const key = inferencePairKey(inference.type, inference.model);
|
|
11773
|
+
if (!pairs.has(key)) pairs.set(key, inference);
|
|
11774
|
+
});
|
|
11775
|
+
return pairs;
|
|
11776
|
+
}
|
|
11683
11777
|
function deriveAgentRoster(config) {
|
|
11684
11778
|
const domain = config.site?.domain;
|
|
11685
11779
|
if (!domain) {
|
|
@@ -11688,8 +11782,8 @@ function deriveAgentRoster(config) {
|
|
|
11688
11782
|
);
|
|
11689
11783
|
}
|
|
11690
11784
|
const roster = /* @__PURE__ */ new Map();
|
|
11691
|
-
|
|
11692
|
-
const key =
|
|
11785
|
+
eachAdmittedInference(config, (inference, jobType) => {
|
|
11786
|
+
const key = inferencePairKey(inference.type, inference.model);
|
|
11693
11787
|
let entry = roster.get(key);
|
|
11694
11788
|
if (!entry) {
|
|
11695
11789
|
entry = {
|
|
@@ -11699,19 +11793,7 @@ function deriveAgentRoster(config) {
|
|
|
11699
11793
|
roster.set(key, entry);
|
|
11700
11794
|
}
|
|
11701
11795
|
if (jobType) entry.servesJobTypes.push(jobType);
|
|
11702
|
-
};
|
|
11703
|
-
for (const jobType of JOB_TYPES) {
|
|
11704
|
-
try {
|
|
11705
|
-
admit(resolveWorkerInference(config, jobType), jobType);
|
|
11706
|
-
} catch {
|
|
11707
|
-
}
|
|
11708
|
-
}
|
|
11709
|
-
for (const actor of ["gatherer", "matcher"]) {
|
|
11710
|
-
try {
|
|
11711
|
-
admit(resolveActorInference(config, actor));
|
|
11712
|
-
} catch {
|
|
11713
|
-
}
|
|
11714
|
-
}
|
|
11796
|
+
});
|
|
11715
11797
|
return [...roster.values()].map(
|
|
11716
11798
|
({ agent, servesJobTypes }) => servesJobTypes.length ? { agent, servesJobTypes } : { agent }
|
|
11717
11799
|
);
|
|
@@ -11719,12 +11801,14 @@ function deriveAgentRoster(config) {
|
|
|
11719
11801
|
|
|
11720
11802
|
// src/browser.ts
|
|
11721
11803
|
var Browser = class {
|
|
11722
|
-
constructor(views, kb, eventBus, project, config, logger) {
|
|
11804
|
+
constructor(views, kb, eventBus, project, config, limitsDiscovery, embeddingProvider, logger) {
|
|
11723
11805
|
this.views = views;
|
|
11724
11806
|
this.kb = kb;
|
|
11725
11807
|
this.eventBus = eventBus;
|
|
11726
11808
|
this.project = project;
|
|
11727
11809
|
this.config = config;
|
|
11810
|
+
this.limitsDiscovery = limitsDiscovery;
|
|
11811
|
+
this.embeddingProvider = embeddingProvider;
|
|
11728
11812
|
this.logger = logger;
|
|
11729
11813
|
}
|
|
11730
11814
|
views;
|
|
@@ -11732,6 +11816,8 @@ var Browser = class {
|
|
|
11732
11816
|
eventBus;
|
|
11733
11817
|
project;
|
|
11734
11818
|
config;
|
|
11819
|
+
limitsDiscovery;
|
|
11820
|
+
embeddingProvider;
|
|
11735
11821
|
subscriptions = [];
|
|
11736
11822
|
logger;
|
|
11737
11823
|
async initialize() {
|
|
@@ -11826,21 +11912,28 @@ var Browser = class {
|
|
|
11826
11912
|
try {
|
|
11827
11913
|
const offset = event.offset ?? 0;
|
|
11828
11914
|
const limit = event.limit ?? 50;
|
|
11829
|
-
const
|
|
11915
|
+
const result = await ResourceContext.listResources({
|
|
11830
11916
|
search: event.search,
|
|
11831
11917
|
archived: event.archived,
|
|
11832
11918
|
entityType: event.entityType,
|
|
11833
11919
|
offset,
|
|
11834
11920
|
limit
|
|
11835
|
-
}, this.kb
|
|
11836
|
-
|
|
11921
|
+
}, this.kb, {
|
|
11922
|
+
embeddingProvider: this.embeddingProvider,
|
|
11923
|
+
semanticFloor: this.config.search.semanticFloor,
|
|
11924
|
+
logger: this.logger
|
|
11925
|
+
});
|
|
11926
|
+
const formattedDocs = event.search && result.matchKind === "lexical" ? await ResourceContext.addContentPreviews(result.resources, this.kb) : result.resources;
|
|
11837
11927
|
this.eventBus.get("browse:resources-result").next({
|
|
11838
11928
|
correlationId: event.correlationId,
|
|
11839
11929
|
response: {
|
|
11840
11930
|
resources: formattedDocs,
|
|
11841
|
-
total,
|
|
11931
|
+
total: result.total,
|
|
11842
11932
|
offset,
|
|
11843
|
-
limit
|
|
11933
|
+
limit,
|
|
11934
|
+
// The producer of the answer labels it (P1b moved the label here
|
|
11935
|
+
// from a hardcoded 'lexical' when the fallback landed).
|
|
11936
|
+
matchKind: result.matchKind
|
|
11844
11937
|
}
|
|
11845
11938
|
});
|
|
11846
11939
|
} catch (error) {
|
|
@@ -12049,7 +12142,7 @@ var Browser = class {
|
|
|
12049
12142
|
}
|
|
12050
12143
|
async handleBrowseAgents(event) {
|
|
12051
12144
|
try {
|
|
12052
|
-
const agents = deriveAgentRoster(this.config);
|
|
12145
|
+
const agents = await this.limitsDiscovery.enrich(deriveAgentRoster(this.config));
|
|
12053
12146
|
this.eventBus.get("browse:agents-result").next({
|
|
12054
12147
|
correlationId: event.correlationId,
|
|
12055
12148
|
response: { agents }
|
|
@@ -12166,6 +12259,63 @@ var Browser = class {
|
|
|
12166
12259
|
this.logger.info("Browser actor stopped");
|
|
12167
12260
|
}
|
|
12168
12261
|
};
|
|
12262
|
+
var LIMITS_ENRICH_BUDGET_MS = 1500;
|
|
12263
|
+
function createLimitsDiscovery(config, logger, options) {
|
|
12264
|
+
const clientFactory = createInferenceClient;
|
|
12265
|
+
const budgetMs = LIMITS_ENRICH_BUDGET_MS;
|
|
12266
|
+
const clients = /* @__PURE__ */ new Map();
|
|
12267
|
+
for (const [pair, inference] of deriveInferencePairs(config)) {
|
|
12268
|
+
try {
|
|
12269
|
+
clients.set(pair, clientFactory(inference, logger));
|
|
12270
|
+
} catch (error) {
|
|
12271
|
+
logger.debug("Limits discovery: client construction failed \u2014 pair enriches as absent", {
|
|
12272
|
+
pair,
|
|
12273
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
12274
|
+
});
|
|
12275
|
+
}
|
|
12276
|
+
}
|
|
12277
|
+
const consult = async (pair, client) => {
|
|
12278
|
+
let timer2;
|
|
12279
|
+
try {
|
|
12280
|
+
const discovery = client.limits();
|
|
12281
|
+
const raced = await Promise.race([
|
|
12282
|
+
discovery,
|
|
12283
|
+
new Promise((resolve2) => {
|
|
12284
|
+
timer2 = setTimeout(() => resolve2(void 0), budgetMs);
|
|
12285
|
+
})
|
|
12286
|
+
]);
|
|
12287
|
+
if (raced === void 0) {
|
|
12288
|
+
void discovery.catch(() => {
|
|
12289
|
+
});
|
|
12290
|
+
logger.debug("Limits discovery: consult exceeded budget \u2014 entry enriches as absent", { pair, budgetMs });
|
|
12291
|
+
return void 0;
|
|
12292
|
+
}
|
|
12293
|
+
return { contextTokens: raced.contextTokens, maxOutputTokens: raced.maxOutputTokens };
|
|
12294
|
+
} catch (error) {
|
|
12295
|
+
logger.debug("Limits discovery: consult failed \u2014 entry enriches as absent", {
|
|
12296
|
+
pair,
|
|
12297
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
12298
|
+
});
|
|
12299
|
+
return void 0;
|
|
12300
|
+
} finally {
|
|
12301
|
+
clearTimeout(timer2);
|
|
12302
|
+
}
|
|
12303
|
+
};
|
|
12304
|
+
return {
|
|
12305
|
+
async enrich(entries) {
|
|
12306
|
+
const settled = await Promise.allSettled(entries.map(async (entry) => {
|
|
12307
|
+
const agent = entry.agent;
|
|
12308
|
+
if (agent["@type"] !== "Software" || !agent.provider || !agent.model) return entry;
|
|
12309
|
+
const pair = inferencePairKey(agent.provider, agent.model);
|
|
12310
|
+
const client = clients.get(pair);
|
|
12311
|
+
if (!client) return entry;
|
|
12312
|
+
const limits = await consult(pair, client);
|
|
12313
|
+
return limits ? { ...entry, limits } : entry;
|
|
12314
|
+
}));
|
|
12315
|
+
return settled.map((s, i) => s.status === "fulfilled" ? s.value : entries[i]);
|
|
12316
|
+
}
|
|
12317
|
+
};
|
|
12318
|
+
}
|
|
12169
12319
|
|
|
12170
12320
|
// src/event-enrichment.ts
|
|
12171
12321
|
function eventAnnotationId(event) {
|
|
@@ -12195,7 +12345,7 @@ function asBusRequestPrimitive(eventBus) {
|
|
|
12195
12345
|
return {
|
|
12196
12346
|
emit(channel, payload) {
|
|
12197
12347
|
eventBus.get(channel).next(payload);
|
|
12198
|
-
return Promise.resolve();
|
|
12348
|
+
return Promise.resolve(-1);
|
|
12199
12349
|
},
|
|
12200
12350
|
stream(channel) {
|
|
12201
12351
|
return eventBus.get(channel).asObservable();
|
|
@@ -12644,6 +12794,38 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12644
12794
|
throw new Error("_userId is required (injected by bus gateway)");
|
|
12645
12795
|
}
|
|
12646
12796
|
const user = parseDidUser(_userId);
|
|
12797
|
+
let effectiveResourceId;
|
|
12798
|
+
if (jobType === "generation") {
|
|
12799
|
+
if (resId !== void 0) {
|
|
12800
|
+
throw new Error(
|
|
12801
|
+
"generation job:create must omit resourceId \u2014 the context's focus is authoritative"
|
|
12802
|
+
);
|
|
12803
|
+
}
|
|
12804
|
+
const bag = params;
|
|
12805
|
+
if (bag && bag.referenceId !== void 0) {
|
|
12806
|
+
throw new Error(
|
|
12807
|
+
"generation job:create must omit params.referenceId \u2014 the context's focus is authoritative"
|
|
12808
|
+
);
|
|
12809
|
+
}
|
|
12810
|
+
if (!isGenerationJobParams(params)) {
|
|
12811
|
+
throw new Error(
|
|
12812
|
+
"generation params do not satisfy GenerationJobParams (title, storageUri, and context are required)"
|
|
12813
|
+
);
|
|
12814
|
+
}
|
|
12815
|
+
const focus = params.context.focus;
|
|
12816
|
+
const rid = focus?.kind === "resource" ? focus.resource?.["@id"] : focus?.kind === "annotation" ? focus.sourceResource?.["@id"] : void 0;
|
|
12817
|
+
if (typeof rid !== "string" || rid.length === 0) {
|
|
12818
|
+
throw new Error(
|
|
12819
|
+
"generation context has no usable focus \u2014 pass a GatheredContext produced by gather.resource(...) or gather.annotation(...)"
|
|
12820
|
+
);
|
|
12821
|
+
}
|
|
12822
|
+
effectiveResourceId = rid;
|
|
12823
|
+
} else {
|
|
12824
|
+
if (typeof resId !== "string" || resId.length === 0) {
|
|
12825
|
+
throw new Error(`${jobType} job:create requires resourceId`);
|
|
12826
|
+
}
|
|
12827
|
+
effectiveResourceId = resId;
|
|
12828
|
+
}
|
|
12647
12829
|
const job = {
|
|
12648
12830
|
status: "pending",
|
|
12649
12831
|
metadata: {
|
|
@@ -12662,7 +12844,7 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12662
12844
|
maxRetries: jobType === "generation" ? 0 : 1
|
|
12663
12845
|
},
|
|
12664
12846
|
params: {
|
|
12665
|
-
resourceId: resourceId(
|
|
12847
|
+
resourceId: resourceId(effectiveResourceId),
|
|
12666
12848
|
...params
|
|
12667
12849
|
}
|
|
12668
12850
|
};
|
|
@@ -12868,35 +13050,40 @@ async function createKnowledgeSystemFromConfig(project, config, eventBus, logger
|
|
|
12868
13050
|
logger.info("Connecting to graph database", { type: graphConfig.type });
|
|
12869
13051
|
const graphDb = await withStartupTimeout("Graph database", getGraphDatabase(graphConfig));
|
|
12870
13052
|
const eventStore = createEventStore(project, eventBus, logger.child({ component: "event-store" }));
|
|
12871
|
-
let vectorStore;
|
|
12872
|
-
let embeddingProvider;
|
|
12873
13053
|
const vectorsConfig = config.services.vectors;
|
|
12874
13054
|
const embeddingConfig = config.services.embedding;
|
|
12875
|
-
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
12879
|
-
|
|
12880
|
-
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
12885
|
-
|
|
12886
|
-
|
|
12887
|
-
|
|
12888
|
-
|
|
12889
|
-
|
|
12890
|
-
|
|
12891
|
-
|
|
12892
|
-
|
|
12893
|
-
|
|
12894
|
-
|
|
12895
|
-
|
|
12896
|
-
})
|
|
12897
|
-
|
|
12898
|
-
|
|
13055
|
+
const { createVectorStore, createEmbeddingProvider } = await import('@semiont/vectors');
|
|
13056
|
+
logger.info("Connecting to embedding provider", { type: embeddingConfig.type, model: embeddingConfig.model });
|
|
13057
|
+
const embeddingProvider = await withStartupTimeout(
|
|
13058
|
+
"Embedding provider",
|
|
13059
|
+
createEmbeddingProvider(embeddingConfig)
|
|
13060
|
+
);
|
|
13061
|
+
logger.info("Connecting to vector store", { type: vectorsConfig.type });
|
|
13062
|
+
const vectorStore = await withStartupTimeout(
|
|
13063
|
+
"Vector store",
|
|
13064
|
+
createVectorStore({
|
|
13065
|
+
type: vectorsConfig.type,
|
|
13066
|
+
host: vectorsConfig.host,
|
|
13067
|
+
port: vectorsConfig.port,
|
|
13068
|
+
// Dimensionality is discovered from the provider, so it is passed as a
|
|
13069
|
+
// thunk rather than probed here: the store calls it only if it needs it
|
|
13070
|
+
// (Qdrant, and only to CREATE a collection). This matches how inference
|
|
13071
|
+
// treats provider-derived facts — the client is built with no I/O and
|
|
13072
|
+
// `limits()` are discovered at the point of use — instead of making a
|
|
13073
|
+
// network round-trip a precondition of booting. A `memory` store, or a
|
|
13074
|
+
// Qdrant whose collections already exist, never consults the provider.
|
|
13075
|
+
dimensions: () => embeddingProvider.dimensions()
|
|
13076
|
+
})
|
|
13077
|
+
);
|
|
13078
|
+
if (vectorsConfig.type === "memory") {
|
|
13079
|
+
logger.info("memory vector store: the index rebuilds from the event log on every restart (reconcile re-embeds)");
|
|
12899
13080
|
}
|
|
13081
|
+
logger.info("Vector search initialized", {
|
|
13082
|
+
store: vectorsConfig.type,
|
|
13083
|
+
embedding: embeddingConfig.type,
|
|
13084
|
+
model: embeddingConfig.model
|
|
13085
|
+
});
|
|
13086
|
+
registerVectorIndexSizeProvider(() => vectorStore.count());
|
|
12900
13087
|
const kb = await createKnowledgeBase(eventStore, project, graphDb, eventBus, logger, {
|
|
12901
13088
|
vectorStore,
|
|
12902
13089
|
skipRebuild
|
|
@@ -12928,7 +13115,8 @@ async function createKnowledgeSystemFromConfig(project, config, eventBus, logger
|
|
|
12928
13115
|
embeddingProvider
|
|
12929
13116
|
);
|
|
12930
13117
|
await matcher.initialize();
|
|
12931
|
-
const
|
|
13118
|
+
const limitsDiscovery = createLimitsDiscovery(config, logger.child({ component: "limits-discovery" }));
|
|
13119
|
+
const browser = new Browser(kb.views, kb, eventBus, project, config, limitsDiscovery, embeddingProvider, logger.child({ component: "browser" }));
|
|
12932
13120
|
await browser.initialize();
|
|
12933
13121
|
const cloneTokenManager = new CloneTokenManager(kb, eventBus, logger.child({ component: "clone-token-manager" }));
|
|
12934
13122
|
await cloneTokenManager.initialize();
|
|
@@ -13011,6 +13199,7 @@ var LocalTransport = class {
|
|
|
13011
13199
|
}
|
|
13012
13200
|
}
|
|
13013
13201
|
);
|
|
13202
|
+
return -1;
|
|
13014
13203
|
}
|
|
13015
13204
|
on(channel, handler) {
|
|
13016
13205
|
const sub = this.bus.get(channel).subscribe(handler);
|
|
@@ -13455,11 +13644,11 @@ var Smelter = class _Smelter {
|
|
|
13455
13644
|
key: checksum,
|
|
13456
13645
|
store: this.anchoredStore
|
|
13457
13646
|
});
|
|
13458
|
-
if ("declined"
|
|
13647
|
+
if (extracted.kind === "declined" || !extracted.items?.length) {
|
|
13459
13648
|
this.logger.info("Re-anchor extraction yielded no geometry", {
|
|
13460
13649
|
resourceId: rid,
|
|
13461
13650
|
contentType,
|
|
13462
|
-
..."declined"
|
|
13651
|
+
...extracted.kind === "declined" ? { declined: extracted.declined } : {}
|
|
13463
13652
|
});
|
|
13464
13653
|
return;
|
|
13465
13654
|
}
|
|
@@ -13494,7 +13683,7 @@ var Smelter = class _Smelter {
|
|
|
13494
13683
|
key: checksum,
|
|
13495
13684
|
store: this.anchoredStore
|
|
13496
13685
|
});
|
|
13497
|
-
if ("declined"
|
|
13686
|
+
if (extracted.kind === "declined") {
|
|
13498
13687
|
this.logger.debug("Extractor declined", { resourceId: resourceId10, contentType, reason: extracted.declined });
|
|
13499
13688
|
return { kind: "skipped", checksum, reason: extracted.declined };
|
|
13500
13689
|
}
|
|
@@ -14238,7 +14427,6 @@ async function replayEvent(event, eventBus, eventStore, resolveBlob, contentStor
|
|
|
14238
14427
|
break;
|
|
14239
14428
|
// Job events are transient — skip during replay
|
|
14240
14429
|
case "job:started":
|
|
14241
|
-
case "job:progress":
|
|
14242
14430
|
case "job:completed":
|
|
14243
14431
|
case "job:failed":
|
|
14244
14432
|
logger?.debug("Skipping job event during replay", { type: event.type });
|