@fortemi/core 2026.6.9 → 2026.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -13
- package/dist/aiwg-index.d.ts +156 -8
- package/dist/aiwg-index.js +479 -28
- package/dist/aiwg-index.js.map +1 -1
- package/dist/index.d.ts +14 -7
- package/dist/index.js +491 -35
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/aiwg-index.js
CHANGED
|
@@ -15,8 +15,6 @@ var REQUIRED_RECORD_FIELDS = [
|
|
|
15
15
|
"id",
|
|
16
16
|
"type",
|
|
17
17
|
"source",
|
|
18
|
-
"title",
|
|
19
|
-
"text",
|
|
20
18
|
"facets",
|
|
21
19
|
"tags",
|
|
22
20
|
"concepts",
|
|
@@ -25,19 +23,14 @@ var REQUIRED_RECORD_FIELDS = [
|
|
|
25
23
|
"privacy",
|
|
26
24
|
"updated_at"
|
|
27
25
|
];
|
|
28
|
-
var VALID_TYPES = /* @__PURE__ */ new Set([
|
|
29
|
-
"crm.contact",
|
|
30
|
-
"crm.organization",
|
|
31
|
-
"crm.event",
|
|
32
|
-
"crm.interaction",
|
|
33
|
-
"aiwg.artifact",
|
|
34
|
-
"docs.page"
|
|
35
|
-
]);
|
|
36
26
|
var DEFAULT_QUERY_WEIGHTS = {
|
|
37
27
|
title: 4,
|
|
38
28
|
tag: 3,
|
|
39
29
|
concept: 2,
|
|
40
|
-
text: 1
|
|
30
|
+
text: 1,
|
|
31
|
+
facet: 2,
|
|
32
|
+
id: 1,
|
|
33
|
+
source: 0.25
|
|
41
34
|
};
|
|
42
35
|
function hasString(value) {
|
|
43
36
|
return typeof value === "string" && value.length > 0;
|
|
@@ -62,6 +55,12 @@ function isPlainRecord(value) {
|
|
|
62
55
|
function isOptionalStringArray(value) {
|
|
63
56
|
return value === void 0 || Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
64
57
|
}
|
|
58
|
+
function isSupportedIndexSchemaVersion(value) {
|
|
59
|
+
return value === "aiwg.fortemi.index.export.v1" || value === "aiwg.fortemi.index.export.v2";
|
|
60
|
+
}
|
|
61
|
+
function isSupportedRecordSchemaVersion(value) {
|
|
62
|
+
return value === "aiwg.fortemi.index.record.v1" || value === "aiwg.fortemi.index.record.v2";
|
|
63
|
+
}
|
|
65
64
|
function validateOptionalRichMetadata(item, index, errors) {
|
|
66
65
|
if (item.skos_concepts !== void 0) {
|
|
67
66
|
if (!Array.isArray(item.skos_concepts)) {
|
|
@@ -108,28 +107,80 @@ function validateOptionalRichMetadata(item, index, errors) {
|
|
|
108
107
|
if (relationship.metadata !== void 0 && !isPlainRecord(relationship.metadata)) {
|
|
109
108
|
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].metadata must be an object");
|
|
110
109
|
}
|
|
110
|
+
if (relationship.direction !== void 0 && relationship.direction !== "upstream" && relationship.direction !== "downstream" && relationship.direction !== "related") {
|
|
111
|
+
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].direction must be upstream, downstream, or related");
|
|
112
|
+
}
|
|
113
|
+
if (relationship.target_path !== void 0 && typeof relationship.target_path !== "string") {
|
|
114
|
+
errors.push("items[" + index + "].relationships[" + relationshipIndex + "].target_path must be a string");
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (item.search !== void 0) {
|
|
119
|
+
if (!isPlainRecord(item.search)) {
|
|
120
|
+
errors.push("items[" + index + "].search must be an object");
|
|
121
|
+
} else {
|
|
122
|
+
if (!isOptionalStringArray(item.search.triggers)) errors.push("items[" + index + "].search.triggers must be a string array");
|
|
123
|
+
if (!isOptionalStringArray(item.search.aliases)) errors.push("items[" + index + "].search.aliases must be a string array");
|
|
124
|
+
if (!isOptionalStringArray(item.search.tags)) errors.push("items[" + index + "].search.tags must be a string array");
|
|
125
|
+
if (item.search.frontmatter !== void 0 && !isPlainRecord(item.search.frontmatter)) {
|
|
126
|
+
errors.push("items[" + index + "].search.frontmatter must be an object");
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (item.chunks !== void 0) {
|
|
131
|
+
if (!Array.isArray(item.chunks)) {
|
|
132
|
+
errors.push("items[" + index + "].chunks must be an array when present");
|
|
133
|
+
} else {
|
|
134
|
+
for (const [chunkIndex, chunk] of item.chunks.entries()) {
|
|
135
|
+
if (!isPlainRecord(chunk)) errors.push("items[" + index + "].chunks[" + chunkIndex + "] must be an object");
|
|
136
|
+
if (chunk.metadata !== void 0 && !isPlainRecord(chunk.metadata)) {
|
|
137
|
+
errors.push("items[" + index + "].chunks[" + chunkIndex + "].metadata must be an object");
|
|
138
|
+
}
|
|
139
|
+
}
|
|
111
140
|
}
|
|
112
141
|
}
|
|
142
|
+
if (item.embeddings !== void 0) {
|
|
143
|
+
if (!Array.isArray(item.embeddings)) {
|
|
144
|
+
errors.push("items[" + index + "].embeddings must be an array when present");
|
|
145
|
+
} else {
|
|
146
|
+
for (const [embeddingIndex, embedding] of item.embeddings.entries()) {
|
|
147
|
+
if (!isPlainRecord(embedding)) errors.push("items[" + index + "].embeddings[" + embeddingIndex + "] must be an object");
|
|
148
|
+
const vector = embedding.embedding ?? embedding.vector;
|
|
149
|
+
if (vector !== void 0 && (!Array.isArray(vector) || !vector.every((entry) => typeof entry === "number"))) {
|
|
150
|
+
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].embedding/vector must be a number array");
|
|
151
|
+
}
|
|
152
|
+
if (embedding.metadata !== void 0 && !isPlainRecord(embedding.metadata)) {
|
|
153
|
+
errors.push("items[" + index + "].embeddings[" + embeddingIndex + "].metadata must be an object");
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
if (item.compatibility !== void 0 && !isPlainRecord(item.compatibility)) {
|
|
159
|
+
errors.push("items[" + index + "].compatibility must be an object");
|
|
160
|
+
}
|
|
113
161
|
}
|
|
114
162
|
function validateAiwgFortemiIndexExport(value) {
|
|
115
163
|
const errors = [];
|
|
116
164
|
const counts = {};
|
|
117
165
|
const data = value;
|
|
118
|
-
if (data?.schema_version
|
|
119
|
-
errors.push("schema_version must be aiwg.fortemi.index.export.v1");
|
|
166
|
+
if (!isSupportedIndexSchemaVersion(data?.schema_version)) {
|
|
167
|
+
errors.push("schema_version must be aiwg.fortemi.index.export.v1 or aiwg.fortemi.index.export.v2");
|
|
120
168
|
}
|
|
121
169
|
if (!hasString(data?.generated_at)) errors.push("generated_at is required");
|
|
122
170
|
if (!hasString(data?.source?.repo)) errors.push("source.repo is required");
|
|
123
171
|
if (!hasString(data?.source?.privacy)) errors.push("source.privacy is required");
|
|
124
172
|
if (!Array.isArray(data?.items)) errors.push("items must be an array");
|
|
173
|
+
if (data.compatibility !== void 0 && !isPlainRecord(data.compatibility)) {
|
|
174
|
+
errors.push("compatibility must be an object");
|
|
175
|
+
}
|
|
125
176
|
const ids = /* @__PURE__ */ new Set();
|
|
126
177
|
let previousId = "";
|
|
127
178
|
for (const [index, item] of (data.items ?? []).entries()) {
|
|
128
179
|
for (const field of REQUIRED_RECORD_FIELDS) {
|
|
129
180
|
if (!(field in item)) errors.push("items[" + index + "]." + field + " is required");
|
|
130
181
|
}
|
|
131
|
-
if (item.schema_version
|
|
132
|
-
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1");
|
|
182
|
+
if (!isSupportedRecordSchemaVersion(item.schema_version)) {
|
|
183
|
+
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1 or aiwg.fortemi.index.record.v2");
|
|
133
184
|
}
|
|
134
185
|
if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
|
|
135
186
|
if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
|
|
@@ -138,11 +189,17 @@ function validateAiwgFortemiIndexExport(value) {
|
|
|
138
189
|
errors.push("items must be sorted by id: " + previousId + " before " + item.id);
|
|
139
190
|
}
|
|
140
191
|
if (hasString(item.id)) previousId = item.id;
|
|
141
|
-
if (!
|
|
192
|
+
if (!hasString(item.type)) errors.push("items[" + index + "].type must be a non-empty string");
|
|
142
193
|
else counts[item.type] = (counts[item.type] ?? 0) + 1;
|
|
143
194
|
if (!hasString(item.source?.path)) errors.push("items[" + index + "].source.path is required");
|
|
144
195
|
if (!hasString(item.source?.repo_relative_path)) errors.push("items[" + index + "].source.repo_relative_path is required");
|
|
145
196
|
if (!hasString(item.source?.locator)) errors.push("items[" + index + "].source.locator is required");
|
|
197
|
+
if (typeof item.title !== "string" && typeof item.search?.title !== "string" && typeof item.search?.name !== "string") {
|
|
198
|
+
errors.push("items[" + index + "].title or search.title/search.name is required");
|
|
199
|
+
}
|
|
200
|
+
if (typeof item.text !== "string" && typeof item.search?.body !== "string" && typeof item.search?.summary !== "string") {
|
|
201
|
+
errors.push("items[" + index + "].text or search.body/search.summary is required");
|
|
202
|
+
}
|
|
146
203
|
if (!Array.isArray(item.tags)) errors.push("items[" + index + "].tags must be an array");
|
|
147
204
|
if (!Array.isArray(item.concepts)) errors.push("items[" + index + "].concepts must be an array");
|
|
148
205
|
if (!Array.isArray(item.relationships)) errors.push("items[" + index + "].relationships must be an array");
|
|
@@ -223,8 +280,8 @@ function validateProjectedRecords(items) {
|
|
|
223
280
|
const ids = /* @__PURE__ */ new Set();
|
|
224
281
|
let previousId = "";
|
|
225
282
|
for (const [index, item] of items.entries()) {
|
|
226
|
-
if (item.schema_version
|
|
227
|
-
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1");
|
|
283
|
+
if (!isSupportedRecordSchemaVersion(item.schema_version)) {
|
|
284
|
+
errors.push("items[" + index + "].schema_version must be aiwg.fortemi.index.record.v1 or aiwg.fortemi.index.record.v2");
|
|
228
285
|
}
|
|
229
286
|
if (!hasString(item.id)) errors.push("items[" + index + "].id is required");
|
|
230
287
|
if (hasString(item.id) && ids.has(item.id)) errors.push("duplicate id: " + item.id);
|
|
@@ -233,9 +290,13 @@ function validateProjectedRecords(items) {
|
|
|
233
290
|
errors.push("items must be sorted by id: " + previousId + " before " + item.id);
|
|
234
291
|
}
|
|
235
292
|
if (hasString(item.id)) previousId = item.id;
|
|
236
|
-
if (!
|
|
237
|
-
if (
|
|
238
|
-
|
|
293
|
+
if (!hasString(item.type)) errors.push("items[" + index + "].type must be a non-empty string");
|
|
294
|
+
if (typeof item.title !== "string" && typeof item.search?.title !== "string" && typeof item.search?.name !== "string") {
|
|
295
|
+
errors.push("items[" + index + "].title or search.title/search.name is required");
|
|
296
|
+
}
|
|
297
|
+
if (typeof item.text !== "string" && typeof item.search?.body !== "string" && typeof item.search?.summary !== "string") {
|
|
298
|
+
errors.push("items[" + index + "].text or search.body/search.summary is required");
|
|
299
|
+
}
|
|
239
300
|
if (!item.facets || typeof item.facets !== "object" || Array.isArray(item.facets)) {
|
|
240
301
|
errors.push("items[" + index + "].facets must be an object");
|
|
241
302
|
}
|
|
@@ -327,6 +388,38 @@ function getAiwgFortemiFacets(items) {
|
|
|
327
388
|
}
|
|
328
389
|
return result;
|
|
329
390
|
}
|
|
391
|
+
function recordTitle(item) {
|
|
392
|
+
return item.title ?? item.search?.title ?? item.search?.name ?? item.id;
|
|
393
|
+
}
|
|
394
|
+
function recordText(item) {
|
|
395
|
+
return item.text ?? item.search?.body ?? item.search?.summary ?? item.chunks?.map((chunk) => chunk.text ?? chunk.body ?? chunk.summary ?? "").filter(Boolean).join("\n") ?? "";
|
|
396
|
+
}
|
|
397
|
+
function recordSearchValues(item) {
|
|
398
|
+
const search = item.search;
|
|
399
|
+
const values = [
|
|
400
|
+
item.id,
|
|
401
|
+
recordTitle(item),
|
|
402
|
+
recordText(item),
|
|
403
|
+
search?.title,
|
|
404
|
+
search?.name,
|
|
405
|
+
search?.summary,
|
|
406
|
+
search?.body,
|
|
407
|
+
search?.capability,
|
|
408
|
+
search?.phase,
|
|
409
|
+
search?.type,
|
|
410
|
+
...search?.triggers ?? [],
|
|
411
|
+
...search?.aliases ?? [],
|
|
412
|
+
...search?.tags ?? [],
|
|
413
|
+
...(item.chunks ?? []).flatMap((chunk) => [chunk.text, chunk.body, chunk.summary, chunk.source_path])
|
|
414
|
+
];
|
|
415
|
+
if (search?.frontmatter) {
|
|
416
|
+
for (const value of Object.values(search.frontmatter)) {
|
|
417
|
+
if (typeof value === "string") values.push(value);
|
|
418
|
+
else if (Array.isArray(value)) values.push(...value.filter((entry) => typeof entry === "string"));
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
return values.filter((value) => typeof value === "string" && value.length > 0);
|
|
422
|
+
}
|
|
330
423
|
function buildAiwgChunkedIndex(index, options = {}) {
|
|
331
424
|
const partSize = hasPositiveInteger(options.partSize) ? options.partSize : 500;
|
|
332
425
|
const projection = options.projection;
|
|
@@ -388,18 +481,129 @@ function matchesFacetFilters(item, filters) {
|
|
|
388
481
|
function queryMatches(item, q) {
|
|
389
482
|
if (!q) return [];
|
|
390
483
|
const matches = [];
|
|
391
|
-
|
|
392
|
-
|
|
484
|
+
const title = recordTitle(item);
|
|
485
|
+
const text = recordText(item);
|
|
486
|
+
if (title.toLowerCase().includes(q)) matches.push({ field: "title", value: title });
|
|
487
|
+
if (text.toLowerCase().includes(q)) matches.push({ field: "text", value: text });
|
|
393
488
|
for (const tag of item.tags) {
|
|
394
489
|
if (tag.toLowerCase().includes(q)) matches.push({ field: "tag", value: tag });
|
|
395
490
|
}
|
|
396
491
|
for (const concept of item.concepts) {
|
|
397
492
|
if (concept.toLowerCase().includes(q)) matches.push({ field: "concept", value: concept });
|
|
398
493
|
}
|
|
494
|
+
for (const value of recordSearchValues(item)) {
|
|
495
|
+
if (value !== title && value !== text && value.toLowerCase().includes(q)) {
|
|
496
|
+
matches.push({ field: "text", value, score: DEFAULT_QUERY_WEIGHTS.text });
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
return matches;
|
|
500
|
+
}
|
|
501
|
+
var DISCOVERY_STOPWORDS = /* @__PURE__ */ new Set([
|
|
502
|
+
"a",
|
|
503
|
+
"an",
|
|
504
|
+
"and",
|
|
505
|
+
"are",
|
|
506
|
+
"as",
|
|
507
|
+
"for",
|
|
508
|
+
"from",
|
|
509
|
+
"how",
|
|
510
|
+
"i",
|
|
511
|
+
"in",
|
|
512
|
+
"is",
|
|
513
|
+
"me",
|
|
514
|
+
"of",
|
|
515
|
+
"on",
|
|
516
|
+
"or",
|
|
517
|
+
"please",
|
|
518
|
+
"the",
|
|
519
|
+
"to",
|
|
520
|
+
"use",
|
|
521
|
+
"with"
|
|
522
|
+
]);
|
|
523
|
+
function normalizeDiscoveryText(value) {
|
|
524
|
+
return value.toLowerCase().replace(/[_/]+/g, " ").replace(/[^a-z0-9.-]+/g, " ").trim();
|
|
525
|
+
}
|
|
526
|
+
function canonicalDiscoveryName(value) {
|
|
527
|
+
return normalizeDiscoveryText(value).replace(/[\s.-]+/g, "");
|
|
528
|
+
}
|
|
529
|
+
function discoveryTokens(value) {
|
|
530
|
+
return normalizeDiscoveryText(value).split(/\s+/).filter((token) => token.length > 1 && !DISCOVERY_STOPWORDS.has(token));
|
|
531
|
+
}
|
|
532
|
+
function facetValues(item, names) {
|
|
533
|
+
return names.flatMap((name) => item.facets[name] ?? []);
|
|
534
|
+
}
|
|
535
|
+
function addDiscoveryMatch(matches, match) {
|
|
536
|
+
if (!matches.some((existing) => existing.field === match.field && existing.value === match.value && existing.reason === match.reason)) {
|
|
537
|
+
matches.push(match);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
function tokenOverlapScore(tokens, value) {
|
|
541
|
+
if (tokens.length === 0 || !value) return 0;
|
|
542
|
+
const normalized = normalizeDiscoveryText(value);
|
|
543
|
+
const hits = tokens.filter((token) => normalized.includes(token)).length;
|
|
544
|
+
return hits / tokens.length;
|
|
545
|
+
}
|
|
546
|
+
function discoveryMatches(item, query) {
|
|
547
|
+
if (!query) return [];
|
|
548
|
+
const matches = [];
|
|
549
|
+
const tokens = discoveryTokens(query);
|
|
550
|
+
const canonicalQuery = canonicalDiscoveryName(query);
|
|
551
|
+
const idParts = item.id.split(/[:/]/);
|
|
552
|
+
const names = [
|
|
553
|
+
item.id,
|
|
554
|
+
recordTitle(item),
|
|
555
|
+
item.search?.name,
|
|
556
|
+
item.search?.title,
|
|
557
|
+
...item.search?.aliases ?? [],
|
|
558
|
+
...idParts,
|
|
559
|
+
...facetValues(item, ["name", "canonical_name", "command", "skill", "agent", "rule"])
|
|
560
|
+
].filter((value) => hasString(value));
|
|
561
|
+
const triggers = [
|
|
562
|
+
...facetValues(item, ["trigger", "triggers", "trigger_phrase", "trigger_phrases"]),
|
|
563
|
+
...item.search?.triggers ?? []
|
|
564
|
+
];
|
|
565
|
+
const capabilities = [
|
|
566
|
+
...facetValues(item, ["capability", "capabilities", "summary", "description"]),
|
|
567
|
+
item.search?.capability,
|
|
568
|
+
item.search?.summary,
|
|
569
|
+
item.search?.phase,
|
|
570
|
+
item.search?.type,
|
|
571
|
+
...item.search?.tags ?? [],
|
|
572
|
+
...item.concepts,
|
|
573
|
+
...item.tags
|
|
574
|
+
].filter((value) => hasString(value));
|
|
575
|
+
const sourceValues = [item.source?.path, item.source?.repo_relative_path, item.source?.locator].filter((value) => hasString(value));
|
|
576
|
+
for (const name of names) {
|
|
577
|
+
const canonicalName = canonicalDiscoveryName(name);
|
|
578
|
+
if (!canonicalName) continue;
|
|
579
|
+
if (canonicalName === canonicalQuery) {
|
|
580
|
+
addDiscoveryMatch(matches, { field: "id", value: name, score: 80, reason: "exact canonical name" });
|
|
581
|
+
} else if (canonicalName.includes(canonicalQuery) || canonicalQuery.includes(canonicalName)) {
|
|
582
|
+
addDiscoveryMatch(matches, { field: "id", value: name, score: 48, reason: "near canonical name" });
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
const title = recordTitle(item);
|
|
586
|
+
const titleOverlap = tokenOverlapScore(tokens, title);
|
|
587
|
+
if (titleOverlap > 0) addDiscoveryMatch(matches, { field: "title", value: title, score: 18 * titleOverlap, reason: "title token overlap" });
|
|
588
|
+
for (const trigger of triggers) {
|
|
589
|
+
const overlap = tokenOverlapScore(tokens, trigger);
|
|
590
|
+
if (overlap > 0) addDiscoveryMatch(matches, { field: "facet", value: trigger, score: 34 * overlap, reason: "trigger phrase" });
|
|
591
|
+
}
|
|
592
|
+
for (const capability of capabilities) {
|
|
593
|
+
const overlap = tokenOverlapScore(tokens, capability);
|
|
594
|
+
if (overlap > 0) addDiscoveryMatch(matches, { field: "concept", value: capability, score: 22 * overlap, reason: "capability overlap" });
|
|
595
|
+
}
|
|
596
|
+
const text = recordText(item);
|
|
597
|
+
const textOverlap = tokenOverlapScore(tokens, text);
|
|
598
|
+
if (textOverlap > 0) addDiscoveryMatch(matches, { field: "text", value: text, score: 8 * textOverlap, reason: "body token overlap" });
|
|
599
|
+
for (const source of sourceValues) {
|
|
600
|
+
const overlap = tokenOverlapScore(tokens, source);
|
|
601
|
+
if (overlap > 0) addDiscoveryMatch(matches, { field: "source", value: source, score: 2 * overlap, reason: "path overlap" });
|
|
602
|
+
}
|
|
399
603
|
return matches;
|
|
400
604
|
}
|
|
401
605
|
function rankMatches(matches, weights) {
|
|
402
|
-
return matches.reduce((total, match) => total + weights[match.field], 0);
|
|
606
|
+
return matches.reduce((total, match) => total + (match.score ?? weights[match.field]), 0);
|
|
403
607
|
}
|
|
404
608
|
function clipSnippet(value, q, maxLength) {
|
|
405
609
|
const normalizedLength = Math.max(20, maxLength);
|
|
@@ -419,11 +623,16 @@ function createSnippet(item, matches, q, maxLength) {
|
|
|
419
623
|
const textMatch = matches.find((match) => match.field === "text");
|
|
420
624
|
const titleMatch = matches.find((match) => match.field === "title");
|
|
421
625
|
const firstMatch = textMatch ?? titleMatch ?? matches[0];
|
|
422
|
-
return clipSnippet(firstMatch?.value ?? item
|
|
626
|
+
return clipSnippet(firstMatch?.value ?? recordText(item), q, maxLength);
|
|
423
627
|
}
|
|
424
628
|
function createRankedEntries(items, q, options, ordinalBase = 0) {
|
|
425
629
|
const weights = { ...DEFAULT_QUERY_WEIGHTS, ...options.weights };
|
|
426
|
-
|
|
630
|
+
const profile = options.searchProfile ?? "default";
|
|
631
|
+
return items.map((item, ordinal) => ({
|
|
632
|
+
item,
|
|
633
|
+
ordinal: ordinalBase + ordinal,
|
|
634
|
+
matches: profile === "aiwg-discovery" ? discoveryMatches(item, q) : queryMatches(item, q)
|
|
635
|
+
})).filter(({ item, matches }) => {
|
|
427
636
|
if (q && matches.length === 0) return false;
|
|
428
637
|
if (options.types && !options.types.includes(item.type)) return false;
|
|
429
638
|
if (options.privacy && !options.privacy.includes(item.privacy.classification)) return false;
|
|
@@ -470,7 +679,106 @@ function createQueryResultFromRankedEntries(entries, query, options) {
|
|
|
470
679
|
}
|
|
471
680
|
function queryAiwgFortemiIndex(index, query = "", options = {}) {
|
|
472
681
|
const q = query.trim().toLowerCase();
|
|
473
|
-
|
|
682
|
+
const entries = createRankedEntries(index.items, q, options);
|
|
683
|
+
if (entries.length === 0 && q && options.searchProfile === "aiwg-discovery") {
|
|
684
|
+
const relaxed = discoveryTokens(q).join(" ");
|
|
685
|
+
return createQueryResultFromRankedEntries(createRankedEntries(index.items, relaxed, options), relaxed, options);
|
|
686
|
+
}
|
|
687
|
+
return createQueryResultFromRankedEntries(entries, q, options);
|
|
688
|
+
}
|
|
689
|
+
function cosineSimilarity(left, right) {
|
|
690
|
+
if (left.length !== right.length || left.length === 0) return 0;
|
|
691
|
+
let dot = 0;
|
|
692
|
+
let leftMag = 0;
|
|
693
|
+
let rightMag = 0;
|
|
694
|
+
for (let i = 0; i < left.length; i += 1) {
|
|
695
|
+
const l = left[i];
|
|
696
|
+
const r = right[i];
|
|
697
|
+
dot += l * r;
|
|
698
|
+
leftMag += l * l;
|
|
699
|
+
rightMag += r * r;
|
|
700
|
+
}
|
|
701
|
+
if (leftMag === 0 || rightMag === 0) return 0;
|
|
702
|
+
return dot / (Math.sqrt(leftMag) * Math.sqrt(rightMag));
|
|
703
|
+
}
|
|
704
|
+
function validateAiwgStaticEmbeddingSet(value) {
|
|
705
|
+
const errors = [];
|
|
706
|
+
const data = value;
|
|
707
|
+
if (data?.schema_version !== "aiwg.fortemi.embedding.set.v1") errors.push("schema_version must be aiwg.fortemi.embedding.set.v1");
|
|
708
|
+
if (!hasString(data?.id)) errors.push("id is required");
|
|
709
|
+
if (!hasString(data?.model)) errors.push("model is required");
|
|
710
|
+
if (!hasPositiveInteger(data?.dimensions)) errors.push("dimensions must be a positive integer");
|
|
711
|
+
if (!hasString(data?.generated_at)) errors.push("generated_at is required");
|
|
712
|
+
if (!hasString(data?.granularity)) errors.push("granularity is required");
|
|
713
|
+
if (!Array.isArray(data?.embeddings)) errors.push("embeddings must be an array");
|
|
714
|
+
for (const [index, embedding] of (data.embeddings ?? []).entries()) {
|
|
715
|
+
if (!hasString(embedding.record_id)) errors.push("embeddings[" + index + "].record_id is required");
|
|
716
|
+
if (!hasString(embedding.input_hash)) errors.push("embeddings[" + index + "].input_hash is required");
|
|
717
|
+
if (!Array.isArray(embedding.embedding)) errors.push("embeddings[" + index + "].embedding must be an array");
|
|
718
|
+
else if (hasPositiveInteger(data?.dimensions) && embedding.embedding.length !== data.dimensions) {
|
|
719
|
+
errors.push("embeddings[" + index + "].embedding length must match dimensions");
|
|
720
|
+
} else if (!embedding.embedding.every((number) => typeof number === "number" && Number.isFinite(number))) {
|
|
721
|
+
errors.push("embeddings[" + index + "].embedding must contain finite numbers");
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
return { valid: errors.length === 0, errors };
|
|
725
|
+
}
|
|
726
|
+
function assertAiwgStaticEmbeddingSet(value) {
|
|
727
|
+
const result = validateAiwgStaticEmbeddingSet(value);
|
|
728
|
+
if (!result.valid) throw new Error("Invalid AIWG Fortemi embedding set:\n" + result.errors.join("\n"));
|
|
729
|
+
return value;
|
|
730
|
+
}
|
|
731
|
+
function queryAiwgSemanticIndex(index, embeddingSet, queryEmbedding, options = {}) {
|
|
732
|
+
assertAiwgStaticEmbeddingSet(embeddingSet);
|
|
733
|
+
if (queryEmbedding.length !== embeddingSet.dimensions) throw new Error("query embedding length must match embedding set dimensions");
|
|
734
|
+
const byId = new Map(index.items.map((item) => [item.id, item]));
|
|
735
|
+
const offset = options.offset ?? 0;
|
|
736
|
+
const limit = options.limit ?? 20;
|
|
737
|
+
return embeddingSet.embeddings.map((embedding) => {
|
|
738
|
+
const item = byId.get(embedding.record_id);
|
|
739
|
+
if (!item) return null;
|
|
740
|
+
return { item, embedding, score: cosineSimilarity(queryEmbedding, embedding.embedding) };
|
|
741
|
+
}).filter((result) => result !== null && result.score >= (options.minScore ?? -1)).sort((left, right) => right.score - left.score || left.item.id.localeCompare(right.item.id)).slice(offset, offset + limit);
|
|
742
|
+
}
|
|
743
|
+
function queryAiwgHybridIndex(index, embeddingSet, query, queryEmbedding, options = {}) {
|
|
744
|
+
const lexical = queryAiwgFortemiIndex(index, query, { ...options, rank: true });
|
|
745
|
+
const semantic = queryAiwgSemanticIndex(index, embeddingSet, queryEmbedding, { limit: index.items.length });
|
|
746
|
+
const lexicalWeight = options.lexicalWeight ?? 0.5;
|
|
747
|
+
const semanticWeight = options.semanticWeight ?? 0.5;
|
|
748
|
+
const lexicalScores = new Map(lexical.rankedItems?.map((entry) => [entry.item.id, entry.rank]) ?? []);
|
|
749
|
+
const maxLexical = Math.max(1, ...lexicalScores.values());
|
|
750
|
+
const embeddingById = new Map(semantic.flatMap((entry) => entry.embedding ? [[entry.item.id, entry.embedding]] : []));
|
|
751
|
+
const semanticScores = new Map(semantic.map((entry) => [entry.item.id, entry.score]));
|
|
752
|
+
const ids = /* @__PURE__ */ new Set([...lexicalScores.keys(), ...semanticScores.keys()]);
|
|
753
|
+
const offset = options.offset ?? 0;
|
|
754
|
+
const limit = options.limit ?? 20;
|
|
755
|
+
return [...ids].map((id) => {
|
|
756
|
+
const item = index.items.find((candidate) => candidate.id === id);
|
|
757
|
+
const embedding = embeddingById.get(id);
|
|
758
|
+
if (!item) return null;
|
|
759
|
+
return {
|
|
760
|
+
item,
|
|
761
|
+
...embedding ? { embedding } : {},
|
|
762
|
+
score: (lexicalScores.get(id) ?? 0) / maxLexical * lexicalWeight + (semanticScores.get(id) ?? 0) * semanticWeight
|
|
763
|
+
};
|
|
764
|
+
}).filter((result) => result !== null && result.score >= (options.minScore ?? -1)).sort((left, right) => right.score - left.score || left.item.id.localeCompare(right.item.id)).slice(offset, offset + limit);
|
|
765
|
+
}
|
|
766
|
+
function findAiwgStaticDuplicatePairs(index, embeddingSet, threshold = 0.9) {
|
|
767
|
+
assertAiwgStaticEmbeddingSet(embeddingSet);
|
|
768
|
+
const byId = new Map(index.items.map((item) => [item.id, item]));
|
|
769
|
+
const pairs = [];
|
|
770
|
+
for (let leftIndex = 0; leftIndex < embeddingSet.embeddings.length; leftIndex += 1) {
|
|
771
|
+
for (let rightIndex = leftIndex + 1; rightIndex < embeddingSet.embeddings.length; rightIndex += 1) {
|
|
772
|
+
const leftEmbedding = embeddingSet.embeddings[leftIndex];
|
|
773
|
+
const rightEmbedding = embeddingSet.embeddings[rightIndex];
|
|
774
|
+
const left = byId.get(leftEmbedding.record_id);
|
|
775
|
+
const right = byId.get(rightEmbedding.record_id);
|
|
776
|
+
if (!left || !right) continue;
|
|
777
|
+
const score = cosineSimilarity(leftEmbedding.embedding, rightEmbedding.embedding);
|
|
778
|
+
if (score >= threshold) pairs.push({ left, right, score });
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
return pairs.sort((left, right) => right.score - left.score || left.left.id.localeCompare(right.left.id));
|
|
474
782
|
}
|
|
475
783
|
function chunkPartCacheKey(part) {
|
|
476
784
|
return `${part.offset}:${part.href}`;
|
|
@@ -569,6 +877,107 @@ async function getChunkRecord(runtime, id) {
|
|
|
569
877
|
}
|
|
570
878
|
return record;
|
|
571
879
|
}
|
|
880
|
+
function relationshipTypeFilter(options) {
|
|
881
|
+
return options?.relationshipType ?? options?.type;
|
|
882
|
+
}
|
|
883
|
+
function edgeFromRelationship(sourceId, relationship) {
|
|
884
|
+
return {
|
|
885
|
+
source_id: sourceId,
|
|
886
|
+
target_id: relationship.target_id,
|
|
887
|
+
type: relationship.type,
|
|
888
|
+
...relationship.source_path ? { source_path: relationship.source_path } : {},
|
|
889
|
+
...relationship.target_path ? { target_path: relationship.target_path } : {},
|
|
890
|
+
...relationship.direction ? { direction: relationship.direction } : {}
|
|
891
|
+
};
|
|
892
|
+
}
|
|
893
|
+
function relationshipMatches(edge, options = {}) {
|
|
894
|
+
const type = relationshipTypeFilter(options);
|
|
895
|
+
const direction = options.direction ?? "both";
|
|
896
|
+
if (type && edge.type !== type) return false;
|
|
897
|
+
if (options.relationshipDirection && edge.direction !== options.relationshipDirection) return false;
|
|
898
|
+
if (options.sourceId && edge.source_id !== options.sourceId) return false;
|
|
899
|
+
if (options.targetId && edge.target_id !== options.targetId) return false;
|
|
900
|
+
if (direction === "out" && options.targetId && edge.target_id !== options.targetId) return false;
|
|
901
|
+
if (direction === "in" && options.sourceId && edge.source_id !== options.sourceId) return false;
|
|
902
|
+
return true;
|
|
903
|
+
}
|
|
904
|
+
function nodeSummary(item) {
|
|
905
|
+
return { id: item.id, type: item.type, title: recordTitle(item) };
|
|
906
|
+
}
|
|
907
|
+
function addNode(nodes, item) {
|
|
908
|
+
if (item) nodes.set(item.id, nodeSummary(item));
|
|
909
|
+
}
|
|
910
|
+
function relationshipResultFromRecords(records, options = {}) {
|
|
911
|
+
const byId = new Map(records.map((record) => [record.id, record]));
|
|
912
|
+
const edges = [];
|
|
913
|
+
for (const record of records) {
|
|
914
|
+
for (const relationship of record.relationships ?? []) {
|
|
915
|
+
const edge = edgeFromRelationship(record.id, relationship);
|
|
916
|
+
if (!relationshipMatches(edge, options)) continue;
|
|
917
|
+
edges.push(edge);
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
const limitedEdges = (options.limit ? edges.slice(0, options.limit) : edges).sort((left, right) => left.source_id.localeCompare(right.source_id) || left.target_id.localeCompare(right.target_id) || left.type.localeCompare(right.type));
|
|
921
|
+
const nodes = /* @__PURE__ */ new Map();
|
|
922
|
+
for (const edge of limitedEdges) {
|
|
923
|
+
addNode(nodes, byId.get(edge.source_id));
|
|
924
|
+
addNode(nodes, byId.get(edge.target_id));
|
|
925
|
+
}
|
|
926
|
+
return {
|
|
927
|
+
nodes: [...nodes.values()].sort((left, right) => left.id.localeCompare(right.id)),
|
|
928
|
+
edges: limitedEdges,
|
|
929
|
+
complete: true
|
|
930
|
+
};
|
|
931
|
+
}
|
|
932
|
+
function neighborQueryOptions(id, options = {}) {
|
|
933
|
+
const direction = options.direction ?? "both";
|
|
934
|
+
return {
|
|
935
|
+
...options,
|
|
936
|
+
...direction === "out" ? { sourceId: id } : {},
|
|
937
|
+
...direction === "in" ? { targetId: id } : {}
|
|
938
|
+
};
|
|
939
|
+
}
|
|
940
|
+
function filterNeighborResult(id, result, options = {}) {
|
|
941
|
+
const direction = options.direction ?? "both";
|
|
942
|
+
const edges = result.edges.filter((edge) => {
|
|
943
|
+
if (direction === "out") return edge.source_id === id;
|
|
944
|
+
if (direction === "in") return edge.target_id === id;
|
|
945
|
+
return edge.source_id === id || edge.target_id === id;
|
|
946
|
+
});
|
|
947
|
+
const ids = /* @__PURE__ */ new Set();
|
|
948
|
+
for (const edge of edges) {
|
|
949
|
+
ids.add(edge.source_id);
|
|
950
|
+
ids.add(edge.target_id);
|
|
951
|
+
}
|
|
952
|
+
return {
|
|
953
|
+
...result,
|
|
954
|
+
edges,
|
|
955
|
+
nodes: result.nodes.filter((node) => ids.has(node.id))
|
|
956
|
+
};
|
|
957
|
+
}
|
|
958
|
+
async function recordsFromChunkedRuntime(runtime, onProgress) {
|
|
959
|
+
let scannedParts = 0;
|
|
960
|
+
let fetchedParts = 0;
|
|
961
|
+
const records = [];
|
|
962
|
+
for (const partRef of runtime.manifest.parts) {
|
|
963
|
+
const loaded = await loadChunkPart(runtime, partRef);
|
|
964
|
+
if (loaded.fetched) fetchedParts += 1;
|
|
965
|
+
scannedParts += 1;
|
|
966
|
+
onProgress?.({ phase: "part", done: scannedParts, total: runtime.manifest.parts.length, href: partRef.href });
|
|
967
|
+
for (const item of loaded.part.items) {
|
|
968
|
+
records.push(item.relationships ? item : await getChunkRecord(runtime, item.id));
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
return { records, scannedParts, fetchedParts };
|
|
972
|
+
}
|
|
973
|
+
async function relationshipResultFromChunkedRuntime(runtime, options = {}) {
|
|
974
|
+
const loaded = await recordsFromChunkedRuntime(runtime);
|
|
975
|
+
return {
|
|
976
|
+
...relationshipResultFromRecords(loaded.records, options),
|
|
977
|
+
scannedParts: loaded.scannedParts,
|
|
978
|
+
fetchedParts: loaded.fetchedParts
|
|
979
|
+
};
|
|
980
|
+
}
|
|
572
981
|
async function queryChunkedAiwgFortemiIndex(runtime, query = "", options = {}) {
|
|
573
982
|
const q = query.trim().toLowerCase();
|
|
574
983
|
let scannedParts = 0;
|
|
@@ -749,6 +1158,39 @@ function createAiwgIndexController(initialIndex) {
|
|
|
749
1158
|
if (!found) throw new Error("Record not found: " + id);
|
|
750
1159
|
return found;
|
|
751
1160
|
},
|
|
1161
|
+
async neighbors(id, options) {
|
|
1162
|
+
try {
|
|
1163
|
+
const queryOptions = neighborQueryOptions(id, options);
|
|
1164
|
+
const result = chunked ? await relationshipResultFromChunkedRuntime(chunked, queryOptions) : relationshipResultFromRecords(requireIndex().items, queryOptions);
|
|
1165
|
+
return filterNeighborResult(id, result, options);
|
|
1166
|
+
} catch (err) {
|
|
1167
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
1168
|
+
notify();
|
|
1169
|
+
throw error;
|
|
1170
|
+
}
|
|
1171
|
+
},
|
|
1172
|
+
async relationshipQuery(options) {
|
|
1173
|
+
try {
|
|
1174
|
+
return chunked ? await relationshipResultFromChunkedRuntime(chunked, options) : relationshipResultFromRecords(requireIndex().items, options);
|
|
1175
|
+
} catch (err) {
|
|
1176
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
1177
|
+
notify();
|
|
1178
|
+
throw error;
|
|
1179
|
+
}
|
|
1180
|
+
},
|
|
1181
|
+
async relationshipSet(options) {
|
|
1182
|
+
const [left, right] = await Promise.all([
|
|
1183
|
+
this.neighbors(options.a, options),
|
|
1184
|
+
this.neighbors(options.b, options)
|
|
1185
|
+
]);
|
|
1186
|
+
const leftIds = new Set(left.nodes.map((node) => node.id).filter((id) => id !== options.a));
|
|
1187
|
+
const rightIds = new Set(right.nodes.map((node) => node.id).filter((id) => id !== options.b));
|
|
1188
|
+
let ids;
|
|
1189
|
+
if (options.op === "intersection") ids = [...leftIds].filter((id) => rightIds.has(id));
|
|
1190
|
+
else if (options.op === "difference") ids = [...leftIds].filter((id) => !rightIds.has(id));
|
|
1191
|
+
else ids = [.../* @__PURE__ */ new Set([...leftIds, ...rightIds])];
|
|
1192
|
+
return { op: options.op, ids: ids.sort() };
|
|
1193
|
+
},
|
|
752
1194
|
clearChunkCache() {
|
|
753
1195
|
chunked?.partCache.clear();
|
|
754
1196
|
chunked?.detailCache.clear();
|
|
@@ -759,6 +1201,15 @@ function createAiwgIndexController(initialIndex) {
|
|
|
759
1201
|
toCommunityGraph(options) {
|
|
760
1202
|
return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
|
|
761
1203
|
},
|
|
1204
|
+
async toCommunityGraphChunked(options) {
|
|
1205
|
+
if (!chunked) return aiwgFortemiIndexToCommunityGraph(requireIndex(), options);
|
|
1206
|
+
const loaded = await recordsFromChunkedRuntime(chunked, options?.onProgress);
|
|
1207
|
+
return aiwgFortemiIndexToCommunityGraph({
|
|
1208
|
+
generated_at: chunked.manifest.generated_at,
|
|
1209
|
+
source: chunked.manifest.source,
|
|
1210
|
+
items: loaded.records
|
|
1211
|
+
}, options);
|
|
1212
|
+
},
|
|
762
1213
|
setReviewDecision(input) {
|
|
763
1214
|
const decision = {
|
|
764
1215
|
...input,
|
|
@@ -834,6 +1285,6 @@ function communityIdsFor(item, options) {
|
|
|
834
1285
|
return [`type:${item.type}`];
|
|
835
1286
|
}
|
|
836
1287
|
|
|
837
|
-
export { AIWG_SCAN_REQUIRED_FIELDS, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, buildAiwgChunkedIndex, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, getAiwgFortemiFacets, queryAiwgFortemiIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport };
|
|
1288
|
+
export { AIWG_SCAN_REQUIRED_FIELDS, aiwgDetailHrefForId, aiwgFortemiIndexToCommunityGraph, assertAiwgFortemiChunkManifest, assertAiwgFortemiChunkPart, assertAiwgFortemiIndexExport, assertAiwgStaticEmbeddingSet, buildAiwgChunkedIndex, createAiwgFetchChunkLoader, createAiwgFetchDetailLoader, createAiwgIndexController, createAiwgReviewDecisionExport, encodeAiwgDetailId, findAiwgStaticDuplicatePairs, getAiwgFortemiFacets, queryAiwgFortemiIndex, queryAiwgHybridIndex, queryAiwgSemanticIndex, validateAiwgFortemiChunkManifest, validateAiwgFortemiChunkPart, validateAiwgFortemiIndexExport, validateAiwgStaticEmbeddingSet };
|
|
838
1289
|
//# sourceMappingURL=aiwg-index.js.map
|
|
839
1290
|
//# sourceMappingURL=aiwg-index.js.map
|