@semiont/make-meaning 0.5.26 → 0.5.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.ts +113 -16
- package/dist/index.js +275 -114
- package/dist/index.js.map +1 -1
- package/dist/smelter-main.js +2 -3
- package/dist/smelter-main.js.map +1 -1
- package/package.json +12 -12
package/dist/index.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { STALL_THRESHOLD_MS, FsJobQueue } from '@semiont/jobs';
|
|
2
2
|
import { FilesystemViewStorage, resolveStorageUri, EventQuery, createEventStore } from '@semiont/event-sourcing';
|
|
3
|
-
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, getPrimaryRepresentation, decodeRepresentation, getBodySource, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, baseMediaType, isSupportedMediaType, capabilitiesOf, didToAgent, assembleAnnotation, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, extensionForMediaType, applyBodyOperations, softwareToAgent } from '@semiont/core';
|
|
3
|
+
import { getResourceEntityTypes, getResourceId, getTargetSource, resourceId, getPrimaryRepresentation, decodeRepresentation, getBodySource, getTargetSelector, deriveViews, getTextPositionSelector, annotationId, errField, userId, generateUuid, getExactText, busRequest, cloneToken, baseMediaType, isSupportedMediaType, capabilitiesOf, didToAgent, assembleAnnotation, isGenerationJobParams, jobId, entityType, baseUrl, busLog, BRIDGED_CHANNELS, burstBuffer, textExtractionOf, chunkText, getPrimaryMediaType, extensionForMediaType, applyBodyOperations, softwareToAgent } from '@semiont/core';
|
|
4
4
|
import { recordGatherDegrade, withActorSpan, recordBusEmit, withSpan, SpanKind, registerJobQueueProvider, registerVectorIndexSizeProvider } from '@semiont/observability';
|
|
5
5
|
import { createInferenceClient } from '@semiont/inference';
|
|
6
6
|
import { compareByRecencyThenId, getGraphDatabase } from '@semiont/graph';
|
|
7
7
|
import { WorkingTreeStore, createAnchoredTextStore, deriveStorageUri, anchoredTextStoreOverTransport, EXTRACTORS, calculateChecksum } from '@semiont/content';
|
|
8
8
|
import { getEntityTypes, DEFAULT_ENTITY_TYPES } from '@semiont/ontology';
|
|
9
|
+
import { mergeByResource } from '@semiont/vectors';
|
|
9
10
|
import { promises } from 'fs';
|
|
10
11
|
import * as path3 from 'path';
|
|
11
12
|
import { createGzip, createGunzip } from 'zlib';
|
|
@@ -9905,11 +9906,9 @@ async function createKnowledgeBase(eventStore, project, graphDb, eventBus, logge
|
|
|
9905
9906
|
graph: graphDb,
|
|
9906
9907
|
weaveProgress,
|
|
9907
9908
|
smeltProgress,
|
|
9909
|
+
vectors: options.vectorStore,
|
|
9908
9910
|
projectionsDir: project.projectionsDir
|
|
9909
9911
|
};
|
|
9910
|
-
if (options?.vectorStore) {
|
|
9911
|
-
kb.vectors = options.vectorStore;
|
|
9912
|
-
}
|
|
9913
9912
|
return kb;
|
|
9914
9913
|
}
|
|
9915
9914
|
var PROJECTION_LAG_BACKOFF_MS = [25, 50, 100, 200];
|
|
@@ -10057,7 +10056,19 @@ Format as a simple list, one suggestion per line.`;
|
|
|
10057
10056
|
}
|
|
10058
10057
|
return response.split("\n").map((line) => line.replace(/^[-*•]\s*/, "").trim()).filter((line) => line.length > 0).slice(0, 3);
|
|
10059
10058
|
}
|
|
10060
|
-
|
|
10059
|
+
|
|
10060
|
+
// src/graph-read-grace.ts
|
|
10061
|
+
async function resourceWithViewGrace(kb, rid) {
|
|
10062
|
+
const fromGraph = await kb.graph.getResource(rid).catch(() => null);
|
|
10063
|
+
if (fromGraph) return { resource: fromGraph, laggedBehindView: false };
|
|
10064
|
+
const view = await kb.views.get(rid);
|
|
10065
|
+
if (view) return { resource: view.resource, laggedBehindView: true };
|
|
10066
|
+
return { resource: null, laggedBehindView: false };
|
|
10067
|
+
}
|
|
10068
|
+
|
|
10069
|
+
// src/resource-context.ts
|
|
10070
|
+
var SEMANTIC_OVER_FETCH = 4;
|
|
10071
|
+
var ResourceContext = class _ResourceContext {
|
|
10061
10072
|
/**
|
|
10062
10073
|
* Get resource metadata from view storage
|
|
10063
10074
|
*/
|
|
@@ -10082,21 +10093,67 @@ var ResourceContext = class {
|
|
|
10082
10093
|
* barrier-stamped projection, so an unsearched listing is read-your-writes
|
|
10083
10094
|
* where the graph is only eventually consistent.
|
|
10084
10095
|
*/
|
|
10085
|
-
static async listResources(filters, kb) {
|
|
10096
|
+
static async listResources(filters, kb, semantic) {
|
|
10086
10097
|
const { search: rawSearch, archived, entityType: entityType2, offset = 0, limit = 50 } = filters ?? {};
|
|
10087
10098
|
const search = rawSearch?.trim() || void 0;
|
|
10088
10099
|
if (search) {
|
|
10089
|
-
|
|
10100
|
+
const lexical = await kb.graph.listResources({
|
|
10090
10101
|
search,
|
|
10091
10102
|
archived,
|
|
10092
10103
|
entityTypes: entityType2 ? [entityType2] : void 0,
|
|
10093
10104
|
offset,
|
|
10094
10105
|
limit
|
|
10095
10106
|
});
|
|
10107
|
+
if (lexical.total > 0 || offset > 0) return { ...lexical, matchKind: "lexical" };
|
|
10108
|
+
return _ResourceContext.semanticFallback(search, limit, kb, semantic);
|
|
10096
10109
|
}
|
|
10097
10110
|
const allViews = await kb.views.getAll();
|
|
10098
10111
|
const matches = allViews.map((view) => view.resource).filter((doc) => archived === void 0 || doc.archived === archived).filter((doc) => !entityType2 || getResourceEntityTypes(doc).includes(entityType2)).sort(compareByRecencyThenId);
|
|
10099
|
-
return { resources: matches.slice(offset, offset + limit), total: matches.length };
|
|
10112
|
+
return { resources: matches.slice(offset, offset + limit), total: matches.length, matchKind: "lexical" };
|
|
10113
|
+
}
|
|
10114
|
+
/**
|
|
10115
|
+
* Answer an empty lexical search from the vector index (SEMANTIC-FALLBACK):
|
|
10116
|
+
* embed the query once, fold chunk hits per resource, floor them, and label
|
|
10117
|
+
* the answer 'semantic' so the UI can say "no title matches, but these
|
|
10118
|
+
* documents discuss it".
|
|
10119
|
+
*
|
|
10120
|
+
* Degradation is the contract (axioms S3–S5): unconfigured vectors, an
|
|
10121
|
+
* absent provider, or ANY failure inside the fallback yields the same
|
|
10122
|
+
* empty page the caller already had, labelled 'lexical' — a broken
|
|
10123
|
+
* fallback must never turn a working empty search into an error.
|
|
10124
|
+
*
|
|
10125
|
+
* The floor is applied HERE rather than passed as `scoreThreshold`, so the
|
|
10126
|
+
* below-floor hits exist to be counted — the debug line is the evidence
|
|
10127
|
+
* decision #1's guessed 0.6 gets tuned from.
|
|
10128
|
+
*/
|
|
10129
|
+
static async semanticFallback(search, limit, kb, semantic) {
|
|
10130
|
+
const empty = { resources: [], total: 0, matchKind: "lexical" };
|
|
10131
|
+
try {
|
|
10132
|
+
const embedding = await semantic.embeddingProvider.embed(search);
|
|
10133
|
+
const hits = await kb.vectors.searchResources(embedding, { limit: limit * SEMANTIC_OVER_FETCH });
|
|
10134
|
+
const merged = mergeByResource(hits);
|
|
10135
|
+
const aboveFloor = merged.filter((h) => h.score >= semantic.semanticFloor);
|
|
10136
|
+
semantic.logger.debug("[search FALLBACK] semantic score distribution", {
|
|
10137
|
+
chunkHits: hits.length,
|
|
10138
|
+
resources: merged.length,
|
|
10139
|
+
aboveFloor: aboveFloor.length,
|
|
10140
|
+
belowFloor: merged.length - aboveFloor.length,
|
|
10141
|
+
topScore: merged[0]?.score,
|
|
10142
|
+
bottomScore: merged[merged.length - 1]?.score,
|
|
10143
|
+
floor: semantic.semanticFloor
|
|
10144
|
+
});
|
|
10145
|
+
const resources = [];
|
|
10146
|
+
for (const hit of aboveFloor.slice(0, limit)) {
|
|
10147
|
+
const { resource } = await resourceWithViewGrace(kb, hit.resourceId);
|
|
10148
|
+
if (resource) resources.push({ ...resource, content: hit.text });
|
|
10149
|
+
}
|
|
10150
|
+
return { resources, total: aboveFloor.length, matchKind: "semantic" };
|
|
10151
|
+
} catch (error) {
|
|
10152
|
+
semantic.logger.warn("[search FALLBACK] degraded to the empty lexical page", {
|
|
10153
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
10154
|
+
});
|
|
10155
|
+
return empty;
|
|
10156
|
+
}
|
|
10100
10157
|
}
|
|
10101
10158
|
/**
|
|
10102
10159
|
* Add content previews to resources (for search results)
|
|
@@ -10146,7 +10203,7 @@ var AnnotationContext = class {
|
|
|
10146
10203
|
* @returns Rich context for LLM processing
|
|
10147
10204
|
* @throws Error if annotation or resource not found
|
|
10148
10205
|
*/
|
|
10149
|
-
static async buildLLMContext(annotationId5, resourceId10, kb, options = {}, inferenceClient, logger
|
|
10206
|
+
static async buildLLMContext(annotationId5, resourceId10, kb, embeddingProvider, options = {}, inferenceClient, logger) {
|
|
10150
10207
|
const {
|
|
10151
10208
|
includeSourceContext = true,
|
|
10152
10209
|
includeTargetContext = true,
|
|
@@ -10278,7 +10335,7 @@ Summary:`;
|
|
|
10278
10335
|
}
|
|
10279
10336
|
}
|
|
10280
10337
|
let semanticContext;
|
|
10281
|
-
if (
|
|
10338
|
+
if (sourceContext?.selected) {
|
|
10282
10339
|
try {
|
|
10283
10340
|
const focalEmbedding = await embeddingProvider.embed(sourceContext.selected);
|
|
10284
10341
|
const results = await kb.vectors.searchAnnotations(focalEmbedding, {
|
|
@@ -10602,46 +10659,44 @@ var LLMContext = class {
|
|
|
10602
10659
|
if (options.includeContent) content.related = relatedContent;
|
|
10603
10660
|
let semanticContext;
|
|
10604
10661
|
const vectors = kb.vectors;
|
|
10605
|
-
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
|
|
10609
|
-
|
|
10610
|
-
|
|
10611
|
-
|
|
10612
|
-
|
|
10613
|
-
|
|
10614
|
-
|
|
10615
|
-
|
|
10616
|
-
|
|
10617
|
-
|
|
10618
|
-
|
|
10619
|
-
|
|
10620
|
-
|
|
10621
|
-
|
|
10622
|
-
|
|
10623
|
-
|
|
10624
|
-
|
|
10625
|
-
|
|
10626
|
-
|
|
10627
|
-
|
|
10628
|
-
});
|
|
10629
|
-
}
|
|
10662
|
+
const excludeEntityTypes = options.excludeEntityTypes ?? [];
|
|
10663
|
+
const search = () => vectors.searchByResource(resourceId10, {
|
|
10664
|
+
limit: options.maxResources,
|
|
10665
|
+
scoreThreshold: 0.5,
|
|
10666
|
+
...excludeEntityTypes.length ? { filter: { excludeEntityTypes } } : {}
|
|
10667
|
+
});
|
|
10668
|
+
let matches = await search();
|
|
10669
|
+
if (matches.length === 0) {
|
|
10670
|
+
const contentChecksum = getPrimaryRepresentation(mainDoc)?.checksum;
|
|
10671
|
+
if (contentChecksum) {
|
|
10672
|
+
try {
|
|
10673
|
+
const outcome = await kb.smeltProgress.whenSettled(resourceIdStr, contentChecksum, settleTimeoutMs);
|
|
10674
|
+
if (outcome === "indexed") {
|
|
10675
|
+
matches = await search();
|
|
10676
|
+
}
|
|
10677
|
+
} catch (error) {
|
|
10678
|
+
if (!(error instanceof SmeltProgressTimeout)) throw error;
|
|
10679
|
+
recordGatherDegrade("vectors");
|
|
10680
|
+
logger.warn("[gather DEGRADED] semanticContext absent \u2014 the vector projection did not settle within the barrier", {
|
|
10681
|
+
resourceId: resourceIdStr,
|
|
10682
|
+
contentChecksum,
|
|
10683
|
+
timeoutMs: settleTimeoutMs
|
|
10684
|
+
});
|
|
10630
10685
|
}
|
|
10631
10686
|
}
|
|
10632
|
-
|
|
10633
|
-
|
|
10634
|
-
|
|
10635
|
-
|
|
10636
|
-
|
|
10637
|
-
|
|
10638
|
-
|
|
10639
|
-
|
|
10640
|
-
|
|
10641
|
-
}
|
|
10642
|
-
|
|
10643
|
-
}
|
|
10644
|
-
}
|
|
10687
|
+
}
|
|
10688
|
+
if (matches.length > 0) {
|
|
10689
|
+
semanticContext = {
|
|
10690
|
+
similar: matches.map((m) => ({
|
|
10691
|
+
text: m.text,
|
|
10692
|
+
resourceId: m.resourceId,
|
|
10693
|
+
...m.annotationId ? { annotationId: m.annotationId } : {},
|
|
10694
|
+
score: m.score,
|
|
10695
|
+
...m.entityTypes ? { entityTypes: m.entityTypes } : {},
|
|
10696
|
+
...m.machineRead ? { machineRead: true } : {}
|
|
10697
|
+
})),
|
|
10698
|
+
...excludeEntityTypes.length ? { excludedEntityTypes: excludeEntityTypes } : {}
|
|
10699
|
+
};
|
|
10645
10700
|
}
|
|
10646
10701
|
return {
|
|
10647
10702
|
focus: {
|
|
@@ -10719,10 +10774,10 @@ var Gatherer = class {
|
|
|
10719
10774
|
annotationId(event.annotationId),
|
|
10720
10775
|
resourceId(event.resourceId),
|
|
10721
10776
|
this.kb,
|
|
10777
|
+
this.embeddingProvider,
|
|
10722
10778
|
event.options ?? {},
|
|
10723
10779
|
this.inferenceClient,
|
|
10724
|
-
this.logger
|
|
10725
|
-
this.embeddingProvider
|
|
10780
|
+
this.logger
|
|
10726
10781
|
);
|
|
10727
10782
|
this.eventBus.get("gather:complete").next({
|
|
10728
10783
|
correlationId: event.correlationId,
|
|
@@ -10791,17 +10846,6 @@ var Gatherer = class {
|
|
|
10791
10846
|
// src/matcher.ts
|
|
10792
10847
|
var import_rxjs2 = __toESM(require_cjs());
|
|
10793
10848
|
var import_operators2 = __toESM(require_operators());
|
|
10794
|
-
|
|
10795
|
-
// src/graph-read-grace.ts
|
|
10796
|
-
async function resourceWithViewGrace(kb, rid) {
|
|
10797
|
-
const fromGraph = await kb.graph.getResource(rid).catch(() => null);
|
|
10798
|
-
if (fromGraph) return { resource: fromGraph, laggedBehindView: false };
|
|
10799
|
-
const view = await kb.views.get(rid);
|
|
10800
|
-
if (view) return { resource: view.resource, laggedBehindView: true };
|
|
10801
|
-
return { resource: null, laggedBehindView: false };
|
|
10802
|
-
}
|
|
10803
|
-
|
|
10804
|
-
// src/matcher.ts
|
|
10805
10849
|
var Matcher = class {
|
|
10806
10850
|
constructor(kb, eventBus, logger, inferenceClient, embeddingProvider) {
|
|
10807
10851
|
this.kb = kb;
|
|
@@ -11116,7 +11160,7 @@ For each candidate, output a line with the number and score, like:
|
|
|
11116
11160
|
* from the scorer.
|
|
11117
11161
|
*/
|
|
11118
11162
|
async searchVectors(searchTerm) {
|
|
11119
|
-
if (!
|
|
11163
|
+
if (!searchTerm.trim()) return [];
|
|
11120
11164
|
try {
|
|
11121
11165
|
const embedding = await this.embeddingProvider.embed(searchTerm);
|
|
11122
11166
|
const results = await this.kb.vectors.searchResources(embedding, {
|
|
@@ -11680,6 +11724,29 @@ var JOB_TYPES = [
|
|
|
11680
11724
|
"tag-annotation",
|
|
11681
11725
|
"generation"
|
|
11682
11726
|
];
|
|
11727
|
+
var inferencePairKey = (provider, model) => `${provider} ${model}`;
|
|
11728
|
+
function eachAdmittedInference(config, visit) {
|
|
11729
|
+
for (const jobType of JOB_TYPES) {
|
|
11730
|
+
try {
|
|
11731
|
+
visit(resolveWorkerInference(config, jobType), jobType);
|
|
11732
|
+
} catch {
|
|
11733
|
+
}
|
|
11734
|
+
}
|
|
11735
|
+
for (const actor of ["gatherer", "matcher"]) {
|
|
11736
|
+
try {
|
|
11737
|
+
visit(resolveActorInference(config, actor));
|
|
11738
|
+
} catch {
|
|
11739
|
+
}
|
|
11740
|
+
}
|
|
11741
|
+
}
|
|
11742
|
+
function deriveInferencePairs(config) {
|
|
11743
|
+
const pairs = /* @__PURE__ */ new Map();
|
|
11744
|
+
eachAdmittedInference(config, (inference) => {
|
|
11745
|
+
const key = inferencePairKey(inference.type, inference.model);
|
|
11746
|
+
if (!pairs.has(key)) pairs.set(key, inference);
|
|
11747
|
+
});
|
|
11748
|
+
return pairs;
|
|
11749
|
+
}
|
|
11683
11750
|
function deriveAgentRoster(config) {
|
|
11684
11751
|
const domain = config.site?.domain;
|
|
11685
11752
|
if (!domain) {
|
|
@@ -11688,8 +11755,8 @@ function deriveAgentRoster(config) {
|
|
|
11688
11755
|
);
|
|
11689
11756
|
}
|
|
11690
11757
|
const roster = /* @__PURE__ */ new Map();
|
|
11691
|
-
|
|
11692
|
-
const key =
|
|
11758
|
+
eachAdmittedInference(config, (inference, jobType) => {
|
|
11759
|
+
const key = inferencePairKey(inference.type, inference.model);
|
|
11693
11760
|
let entry = roster.get(key);
|
|
11694
11761
|
if (!entry) {
|
|
11695
11762
|
entry = {
|
|
@@ -11699,19 +11766,7 @@ function deriveAgentRoster(config) {
|
|
|
11699
11766
|
roster.set(key, entry);
|
|
11700
11767
|
}
|
|
11701
11768
|
if (jobType) entry.servesJobTypes.push(jobType);
|
|
11702
|
-
};
|
|
11703
|
-
for (const jobType of JOB_TYPES) {
|
|
11704
|
-
try {
|
|
11705
|
-
admit(resolveWorkerInference(config, jobType), jobType);
|
|
11706
|
-
} catch {
|
|
11707
|
-
}
|
|
11708
|
-
}
|
|
11709
|
-
for (const actor of ["gatherer", "matcher"]) {
|
|
11710
|
-
try {
|
|
11711
|
-
admit(resolveActorInference(config, actor));
|
|
11712
|
-
} catch {
|
|
11713
|
-
}
|
|
11714
|
-
}
|
|
11769
|
+
});
|
|
11715
11770
|
return [...roster.values()].map(
|
|
11716
11771
|
({ agent, servesJobTypes }) => servesJobTypes.length ? { agent, servesJobTypes } : { agent }
|
|
11717
11772
|
);
|
|
@@ -11719,12 +11774,14 @@ function deriveAgentRoster(config) {
|
|
|
11719
11774
|
|
|
11720
11775
|
// src/browser.ts
|
|
11721
11776
|
var Browser = class {
|
|
11722
|
-
constructor(views, kb, eventBus, project, config, logger) {
|
|
11777
|
+
constructor(views, kb, eventBus, project, config, limitsDiscovery, embeddingProvider, logger) {
|
|
11723
11778
|
this.views = views;
|
|
11724
11779
|
this.kb = kb;
|
|
11725
11780
|
this.eventBus = eventBus;
|
|
11726
11781
|
this.project = project;
|
|
11727
11782
|
this.config = config;
|
|
11783
|
+
this.limitsDiscovery = limitsDiscovery;
|
|
11784
|
+
this.embeddingProvider = embeddingProvider;
|
|
11728
11785
|
this.logger = logger;
|
|
11729
11786
|
}
|
|
11730
11787
|
views;
|
|
@@ -11732,6 +11789,8 @@ var Browser = class {
|
|
|
11732
11789
|
eventBus;
|
|
11733
11790
|
project;
|
|
11734
11791
|
config;
|
|
11792
|
+
limitsDiscovery;
|
|
11793
|
+
embeddingProvider;
|
|
11735
11794
|
subscriptions = [];
|
|
11736
11795
|
logger;
|
|
11737
11796
|
async initialize() {
|
|
@@ -11826,21 +11885,28 @@ var Browser = class {
|
|
|
11826
11885
|
try {
|
|
11827
11886
|
const offset = event.offset ?? 0;
|
|
11828
11887
|
const limit = event.limit ?? 50;
|
|
11829
|
-
const
|
|
11888
|
+
const result = await ResourceContext.listResources({
|
|
11830
11889
|
search: event.search,
|
|
11831
11890
|
archived: event.archived,
|
|
11832
11891
|
entityType: event.entityType,
|
|
11833
11892
|
offset,
|
|
11834
11893
|
limit
|
|
11835
|
-
}, this.kb
|
|
11836
|
-
|
|
11894
|
+
}, this.kb, {
|
|
11895
|
+
embeddingProvider: this.embeddingProvider,
|
|
11896
|
+
semanticFloor: this.config.search.semanticFloor,
|
|
11897
|
+
logger: this.logger
|
|
11898
|
+
});
|
|
11899
|
+
const formattedDocs = event.search && result.matchKind === "lexical" ? await ResourceContext.addContentPreviews(result.resources, this.kb) : result.resources;
|
|
11837
11900
|
this.eventBus.get("browse:resources-result").next({
|
|
11838
11901
|
correlationId: event.correlationId,
|
|
11839
11902
|
response: {
|
|
11840
11903
|
resources: formattedDocs,
|
|
11841
|
-
total,
|
|
11904
|
+
total: result.total,
|
|
11842
11905
|
offset,
|
|
11843
|
-
limit
|
|
11906
|
+
limit,
|
|
11907
|
+
// The producer of the answer labels it (P1b moved the label here
|
|
11908
|
+
// from a hardcoded 'lexical' when the fallback landed).
|
|
11909
|
+
matchKind: result.matchKind
|
|
11844
11910
|
}
|
|
11845
11911
|
});
|
|
11846
11912
|
} catch (error) {
|
|
@@ -12049,7 +12115,7 @@ var Browser = class {
|
|
|
12049
12115
|
}
|
|
12050
12116
|
async handleBrowseAgents(event) {
|
|
12051
12117
|
try {
|
|
12052
|
-
const agents = deriveAgentRoster(this.config);
|
|
12118
|
+
const agents = await this.limitsDiscovery.enrich(deriveAgentRoster(this.config));
|
|
12053
12119
|
this.eventBus.get("browse:agents-result").next({
|
|
12054
12120
|
correlationId: event.correlationId,
|
|
12055
12121
|
response: { agents }
|
|
@@ -12166,6 +12232,63 @@ var Browser = class {
|
|
|
12166
12232
|
this.logger.info("Browser actor stopped");
|
|
12167
12233
|
}
|
|
12168
12234
|
};
|
|
12235
|
+
var LIMITS_ENRICH_BUDGET_MS = 1500;
|
|
12236
|
+
function createLimitsDiscovery(config, logger, options) {
|
|
12237
|
+
const clientFactory = createInferenceClient;
|
|
12238
|
+
const budgetMs = LIMITS_ENRICH_BUDGET_MS;
|
|
12239
|
+
const clients = /* @__PURE__ */ new Map();
|
|
12240
|
+
for (const [pair, inference] of deriveInferencePairs(config)) {
|
|
12241
|
+
try {
|
|
12242
|
+
clients.set(pair, clientFactory(inference, logger));
|
|
12243
|
+
} catch (error) {
|
|
12244
|
+
logger.debug("Limits discovery: client construction failed \u2014 pair enriches as absent", {
|
|
12245
|
+
pair,
|
|
12246
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
12247
|
+
});
|
|
12248
|
+
}
|
|
12249
|
+
}
|
|
12250
|
+
const consult = async (pair, client) => {
|
|
12251
|
+
let timer2;
|
|
12252
|
+
try {
|
|
12253
|
+
const discovery = client.limits();
|
|
12254
|
+
const raced = await Promise.race([
|
|
12255
|
+
discovery,
|
|
12256
|
+
new Promise((resolve2) => {
|
|
12257
|
+
timer2 = setTimeout(() => resolve2(void 0), budgetMs);
|
|
12258
|
+
})
|
|
12259
|
+
]);
|
|
12260
|
+
if (raced === void 0) {
|
|
12261
|
+
void discovery.catch(() => {
|
|
12262
|
+
});
|
|
12263
|
+
logger.debug("Limits discovery: consult exceeded budget \u2014 entry enriches as absent", { pair, budgetMs });
|
|
12264
|
+
return void 0;
|
|
12265
|
+
}
|
|
12266
|
+
return { contextTokens: raced.contextTokens, maxOutputTokens: raced.maxOutputTokens };
|
|
12267
|
+
} catch (error) {
|
|
12268
|
+
logger.debug("Limits discovery: consult failed \u2014 entry enriches as absent", {
|
|
12269
|
+
pair,
|
|
12270
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
12271
|
+
});
|
|
12272
|
+
return void 0;
|
|
12273
|
+
} finally {
|
|
12274
|
+
clearTimeout(timer2);
|
|
12275
|
+
}
|
|
12276
|
+
};
|
|
12277
|
+
return {
|
|
12278
|
+
async enrich(entries) {
|
|
12279
|
+
const settled = await Promise.allSettled(entries.map(async (entry) => {
|
|
12280
|
+
const agent = entry.agent;
|
|
12281
|
+
if (agent["@type"] !== "Software" || !agent.provider || !agent.model) return entry;
|
|
12282
|
+
const pair = inferencePairKey(agent.provider, agent.model);
|
|
12283
|
+
const client = clients.get(pair);
|
|
12284
|
+
if (!client) return entry;
|
|
12285
|
+
const limits = await consult(pair, client);
|
|
12286
|
+
return limits ? { ...entry, limits } : entry;
|
|
12287
|
+
}));
|
|
12288
|
+
return settled.map((s, i) => s.status === "fulfilled" ? s.value : entries[i]);
|
|
12289
|
+
}
|
|
12290
|
+
};
|
|
12291
|
+
}
|
|
12169
12292
|
|
|
12170
12293
|
// src/event-enrichment.ts
|
|
12171
12294
|
function eventAnnotationId(event) {
|
|
@@ -12195,7 +12318,7 @@ function asBusRequestPrimitive(eventBus) {
|
|
|
12195
12318
|
return {
|
|
12196
12319
|
emit(channel, payload) {
|
|
12197
12320
|
eventBus.get(channel).next(payload);
|
|
12198
|
-
return Promise.resolve();
|
|
12321
|
+
return Promise.resolve(-1);
|
|
12199
12322
|
},
|
|
12200
12323
|
stream(channel) {
|
|
12201
12324
|
return eventBus.get(channel).asObservable();
|
|
@@ -12644,6 +12767,38 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12644
12767
|
throw new Error("_userId is required (injected by bus gateway)");
|
|
12645
12768
|
}
|
|
12646
12769
|
const user = parseDidUser(_userId);
|
|
12770
|
+
let effectiveResourceId;
|
|
12771
|
+
if (jobType === "generation") {
|
|
12772
|
+
if (resId !== void 0) {
|
|
12773
|
+
throw new Error(
|
|
12774
|
+
"generation job:create must omit resourceId \u2014 the context's focus is authoritative"
|
|
12775
|
+
);
|
|
12776
|
+
}
|
|
12777
|
+
const bag = params;
|
|
12778
|
+
if (bag && bag.referenceId !== void 0) {
|
|
12779
|
+
throw new Error(
|
|
12780
|
+
"generation job:create must omit params.referenceId \u2014 the context's focus is authoritative"
|
|
12781
|
+
);
|
|
12782
|
+
}
|
|
12783
|
+
if (!isGenerationJobParams(params)) {
|
|
12784
|
+
throw new Error(
|
|
12785
|
+
"generation params do not satisfy GenerationJobParams (title, storageUri, and context are required)"
|
|
12786
|
+
);
|
|
12787
|
+
}
|
|
12788
|
+
const focus = params.context.focus;
|
|
12789
|
+
const rid = focus?.kind === "resource" ? focus.resource?.["@id"] : focus?.kind === "annotation" ? focus.sourceResource?.["@id"] : void 0;
|
|
12790
|
+
if (typeof rid !== "string" || rid.length === 0) {
|
|
12791
|
+
throw new Error(
|
|
12792
|
+
"generation context has no usable focus \u2014 pass a GatheredContext produced by gather.resource(...) or gather.annotation(...)"
|
|
12793
|
+
);
|
|
12794
|
+
}
|
|
12795
|
+
effectiveResourceId = rid;
|
|
12796
|
+
} else {
|
|
12797
|
+
if (typeof resId !== "string" || resId.length === 0) {
|
|
12798
|
+
throw new Error(`${jobType} job:create requires resourceId`);
|
|
12799
|
+
}
|
|
12800
|
+
effectiveResourceId = resId;
|
|
12801
|
+
}
|
|
12647
12802
|
const job = {
|
|
12648
12803
|
status: "pending",
|
|
12649
12804
|
metadata: {
|
|
@@ -12662,7 +12817,7 @@ function registerJobCommandHandlers(eventBus, jobQueue, project, parentLogger) {
|
|
|
12662
12817
|
maxRetries: jobType === "generation" ? 0 : 1
|
|
12663
12818
|
},
|
|
12664
12819
|
params: {
|
|
12665
|
-
resourceId: resourceId(
|
|
12820
|
+
resourceId: resourceId(effectiveResourceId),
|
|
12666
12821
|
...params
|
|
12667
12822
|
}
|
|
12668
12823
|
};
|
|
@@ -12868,35 +13023,40 @@ async function createKnowledgeSystemFromConfig(project, config, eventBus, logger
|
|
|
12868
13023
|
logger.info("Connecting to graph database", { type: graphConfig.type });
|
|
12869
13024
|
const graphDb = await withStartupTimeout("Graph database", getGraphDatabase(graphConfig));
|
|
12870
13025
|
const eventStore = createEventStore(project, eventBus, logger.child({ component: "event-store" }));
|
|
12871
|
-
let vectorStore;
|
|
12872
|
-
let embeddingProvider;
|
|
12873
13026
|
const vectorsConfig = config.services.vectors;
|
|
12874
13027
|
const embeddingConfig = config.services.embedding;
|
|
12875
|
-
|
|
12876
|
-
|
|
12877
|
-
|
|
12878
|
-
|
|
12879
|
-
|
|
12880
|
-
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
|
|
12884
|
-
|
|
12885
|
-
|
|
12886
|
-
|
|
12887
|
-
|
|
12888
|
-
|
|
12889
|
-
|
|
12890
|
-
|
|
12891
|
-
|
|
12892
|
-
|
|
12893
|
-
|
|
12894
|
-
|
|
12895
|
-
|
|
12896
|
-
})
|
|
12897
|
-
|
|
12898
|
-
|
|
13028
|
+
const { createVectorStore, createEmbeddingProvider } = await import('@semiont/vectors');
|
|
13029
|
+
logger.info("Connecting to embedding provider", { type: embeddingConfig.type, model: embeddingConfig.model });
|
|
13030
|
+
const embeddingProvider = await withStartupTimeout(
|
|
13031
|
+
"Embedding provider",
|
|
13032
|
+
createEmbeddingProvider(embeddingConfig)
|
|
13033
|
+
);
|
|
13034
|
+
logger.info("Connecting to vector store", { type: vectorsConfig.type });
|
|
13035
|
+
const vectorStore = await withStartupTimeout(
|
|
13036
|
+
"Vector store",
|
|
13037
|
+
createVectorStore({
|
|
13038
|
+
type: vectorsConfig.type,
|
|
13039
|
+
host: vectorsConfig.host,
|
|
13040
|
+
port: vectorsConfig.port,
|
|
13041
|
+
// Dimensionality is discovered from the provider, so it is passed as a
|
|
13042
|
+
// thunk rather than probed here: the store calls it only if it needs it
|
|
13043
|
+
// (Qdrant, and only to CREATE a collection). This matches how inference
|
|
13044
|
+
// treats provider-derived facts — the client is built with no I/O and
|
|
13045
|
+
// `limits()` are discovered at the point of use — instead of making a
|
|
13046
|
+
// network round-trip a precondition of booting. A `memory` store, or a
|
|
13047
|
+
// Qdrant whose collections already exist, never consults the provider.
|
|
13048
|
+
dimensions: () => embeddingProvider.dimensions()
|
|
13049
|
+
})
|
|
13050
|
+
);
|
|
13051
|
+
if (vectorsConfig.type === "memory") {
|
|
13052
|
+
logger.info("memory vector store: the index rebuilds from the event log on every restart (reconcile re-embeds)");
|
|
12899
13053
|
}
|
|
13054
|
+
logger.info("Vector search initialized", {
|
|
13055
|
+
store: vectorsConfig.type,
|
|
13056
|
+
embedding: embeddingConfig.type,
|
|
13057
|
+
model: embeddingConfig.model
|
|
13058
|
+
});
|
|
13059
|
+
registerVectorIndexSizeProvider(() => vectorStore.count());
|
|
12900
13060
|
const kb = await createKnowledgeBase(eventStore, project, graphDb, eventBus, logger, {
|
|
12901
13061
|
vectorStore,
|
|
12902
13062
|
skipRebuild
|
|
@@ -12928,7 +13088,8 @@ async function createKnowledgeSystemFromConfig(project, config, eventBus, logger
|
|
|
12928
13088
|
embeddingProvider
|
|
12929
13089
|
);
|
|
12930
13090
|
await matcher.initialize();
|
|
12931
|
-
const
|
|
13091
|
+
const limitsDiscovery = createLimitsDiscovery(config, logger.child({ component: "limits-discovery" }));
|
|
13092
|
+
const browser = new Browser(kb.views, kb, eventBus, project, config, limitsDiscovery, embeddingProvider, logger.child({ component: "browser" }));
|
|
12932
13093
|
await browser.initialize();
|
|
12933
13094
|
const cloneTokenManager = new CloneTokenManager(kb, eventBus, logger.child({ component: "clone-token-manager" }));
|
|
12934
13095
|
await cloneTokenManager.initialize();
|
|
@@ -13011,6 +13172,7 @@ var LocalTransport = class {
|
|
|
13011
13172
|
}
|
|
13012
13173
|
}
|
|
13013
13174
|
);
|
|
13175
|
+
return -1;
|
|
13014
13176
|
}
|
|
13015
13177
|
on(channel, handler) {
|
|
13016
13178
|
const sub = this.bus.get(channel).subscribe(handler);
|
|
@@ -14238,7 +14400,6 @@ async function replayEvent(event, eventBus, eventStore, resolveBlob, contentStor
|
|
|
14238
14400
|
break;
|
|
14239
14401
|
// Job events are transient — skip during replay
|
|
14240
14402
|
case "job:started":
|
|
14241
|
-
case "job:progress":
|
|
14242
14403
|
case "job:completed":
|
|
14243
14404
|
case "job:failed":
|
|
14244
14405
|
logger?.debug("Skipping job event during replay", { type: event.type });
|