fawn-memory 0.0.3

package/dist/knowpro/semanticRef.js

@@ -0,0 +1,62 @@
+export function populateSemanticRef(semanticRefs, knowledgeResponse, baseSemanticRefOrdinal, messageOrdinal) {
+    let semanticRefOrdinalModifier = baseSemanticRefOrdinal;
+    // create semantic refs from the extracted knowledge here
+    try {
+        knowledgeResponse.entities.forEach((entity) => {
+            const newSemanticRef = {
+                semanticRefOrdinal: semanticRefOrdinalModifier,
+                knowledgeType: "entity",
+                range: {
+                    start: {
+                        messageOrdinal
+                    },
+                    end: {
+                        messageOrdinal
+                    }
+                },
+                knowledge: entity
+            };
+            semanticRefs.append(newSemanticRef);
+            semanticRefOrdinalModifier += 1;
+        });
+        knowledgeResponse.actions.forEach((action) => {
+            const newSemanticRef = {
+                semanticRefOrdinal: semanticRefOrdinalModifier,
+                knowledgeType: "action",
+                knowledge: action,
+                range: {
+                    start: {
+                        messageOrdinal
+                    },
+                    end: {
+                        messageOrdinal
+                    }
+                },
+            };
+            semanticRefs.append(newSemanticRef);
+            semanticRefOrdinalModifier += 1;
+        });
+        knowledgeResponse.topics.forEach((topic) => {
+            const newSemanticRef = {
+                semanticRefOrdinal: semanticRefOrdinalModifier,
+                knowledgeType: "topic",
+                knowledge: { text: topic },
+                range: {
+                    start: {
+                        messageOrdinal
+                    },
+                    end: {
+                        messageOrdinal
+                    }
+                },
+            };
+            semanticRefs.append(newSemanticRef);
+            semanticRefOrdinalModifier += 1;
+        });
+    }
+    catch (error) {
+        console.log(error);
+    }
+    return semanticRefs;
+}
+//# sourceMappingURL=semanticRef.js.map

package/dist/knowpro/semanticRef.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semanticRef.js","sourceRoot":"","sources":["../../src/knowpro/semanticRef.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,mBAAmB,CAC/B,YAAoC,EACpC,iBAAoC,EACpC,sBAA8B,EAC9B,cAAsB;IAEtB,IAAI,0BAA0B,GAAG,sBAAsB,CAAC;IACxD,yDAAyD;IACzD,IAAI,CAAC;QAED,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC1C,MAAM,cAAc,GAAgB;gBAChC,kBAAkB,EAAE,0BAA0B;gBAC9C,aAAa,EAAE,QAAQ;gBACvB,KAAK,EAAE;oBACH,KAAK,EAAE;wBACH,cAAc;qBACjB;oBACD,GAAG,EAAE;wBACD,cAAc;qBACjB;iBACJ;gBACD,SAAS,EAAE,MAAM;aACpB,CAAA;YACD,YAAY,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YACpC,0BAA0B,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACzC,MAAM,cAAc,GAAgB;gBAChC,kBAAkB,EAAE,0BAA0B;gBAC9C,aAAa,EAAE,QAAQ;gBACvB,SAAS,EAAE,MAAM;gBACjB,KAAK,EAAE;oBACH,KAAK,EAAE;wBACH,cAAc;qBACjB;oBACD,GAAG,EAAE;wBACD,cAAc;qBACjB;iBACJ;aACJ,CAAA;YACD,YAAY,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YACpC,0BAA0B,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACvC,MAAM,cAAc,GAAgB;gBAChC,kBAAkB,EAAE,0BAA0B;gBAC9C,aAAa,EAAE,OAAO;gBACtB,SAAS,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;gBAC1B,KAAK,EAAE;oBACH,KAAK,EAAE;wBACH,cAAc;qBACjB;oBACD,GAAG,EAAE;wBACD,cAAc;qBACjB;iBACJ;aACJ,CAAA;YACD,YAAY,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YACpC,0BAA0B,IAAI,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IAEP,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC;IAED,OAAO,YAAY,CAAC;AACxB,CAAC"}

package/dist/knowpro/semanticRefIndex.d.ts

@@ -0,0 +1,48 @@
+import type { ISemanticRefCollection, ITermToSemanticRefIndex, ScoredSemanticRefOrdinal, SemanticRefOrdinal, Topic } from "./interfaces.js";
+import type { Action, ConcreteEntity, Facet } from "./knowledgeSchema.js";
+import type { TypeChatLanguageModel } from "typechat";
+export interface RelatedTermsExpansionOptions {
+    /** Weight for related terms (0-1, default 0.9) */
+    relatedTermWeight?: number;
+    /** Batch size as percentage of total terms (0-1, default 0.1 = 10%) */
+    batchSizePercent?: number;
+    /** Maximum terms per batch (default 50) */
+    maxBatchSize?: number;
+    /** Minimum terms per batch (default 5) */
+    minBatchSize?: number;
+    /** Maximum total terms to process (default: all eligible terms) */
+    maxTerms?: number;
+}
+export interface RelatedTermsExpansionResult {
+    termsProcessed: number;
+    relatedTermsAdded: number;
+    /** Map of original term -> related terms added */
+    expansionMap: Map<string, string[]>;
+}
+export interface WordNetExpansionOptions {
+    /** Weight for related terms (0-1, default 0.9) */
+    relatedTermWeight?: number;
+    /** Maximum total terms to process (default: all eligible terms) */
+    maxTerms?: number;
+}
+export interface InMemorySemanticRefIndex extends ITermToSemanticRefIndex {
+    addEntity: (entity: ConcreteEntity, refOrdinal: SemanticRefOrdinal) => void;
+    addFacet: (facet: Facet, refOrdinal: SemanticRefOrdinal) => void;
+    addAction: (action: Action, refOrdinal: SemanticRefOrdinal) => void;
+    addTopic: (topic: Topic, refOrdinal: SemanticRefOrdinal) => void;
+    buildIndex: (semanticRefCollection: ISemanticRefCollection) => void;
+    lemmatizeIndex: () => LemmatizationResult;
+    expandWithRelatedTerms: (model: TypeChatLanguageModel, options?: RelatedTermsExpansionOptions) => Promise<RelatedTermsExpansionResult>;
+    expandWithWordNet: (options?: WordNetExpansionOptions) => Promise<RelatedTermsExpansionResult>;
+    loadFromExport: (items: Array<{
+        term: string;
+        semanticRefOrdinals: ScoredSemanticRefOrdinal[];
+    }>) => void;
+}
+export interface LemmatizationResult {
+    termsProcessed: number;
+    lemmasAdded: number;
+    lemmaMap: Map<string, string>;
+}
+export declare function createInMemorySemanticRefIndex(): InMemorySemanticRefIndex;
+//# sourceMappingURL=semanticRefIndex.d.ts.map

package/dist/knowpro/semanticRefIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semanticRefIndex.d.ts","sourceRoot":"","sources":["../../src/knowpro/semanticRefIndex.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,uBAAuB,EAAE,wBAAwB,EAAe,kBAAkB,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACzJ,OAAO,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE1E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAOtD,MAAM,WAAW,4BAA4B;IACzC,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2CAA2C;IAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,2BAA2B;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kDAAkD;IAClD,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,uBAAuB;IACpC,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mEAAmE;IACnE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,wBAAyB,SAAQ,uBAAuB;IACrE,SAAS,EAAE,CAAC,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,kBAAkB,KAAK,IAAI,CAAC;IAC5E,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACjE,SAAS,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACpE,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACjE,UAAU,EAAE,CAAC,qBAAqB,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACpE,cAAc,EAAE,MAAM,mBAAmB,CAAC;IAC1C,sBAAsB,EAAE,CAAC,KAAK,EAAE,qBAAqB,EAAE,OAAO,CAAC,EAAE,4BAA4B,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;IACvI,iBAAiB,EAAE,CAAC,OAAO,CAAC,EAAE,uBAAuB,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAC/F,cAAc,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,mBAAmB,EAAE,wBAAwB,EAAE,CAAA;KAAE,CAAC,KAAK,IAAI,CAAC;CAC7G;AAED,MAAM,WAAW,mBAAmB;IAChC,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,wBAAgB,8BAA8B,IAAI,wBAAwB,CA6czE"}

package/dist/knowpro/semanticRefIndex.js

@@ -0,0 +1,388 @@
+import lemmatizer from "wink-lemmatizer";
+import { createJsonTranslator } from "typechat";
+import { createTypeScriptJsonValidator } from "typechat/ts";
+import { loadSchema } from "./schema.js";
+import natural from "natural";
+export function createInMemorySemanticRefIndex() {
+    const map = new Map();
+    return {
+        getTerms,
+        addTerm,
+        removeTerm,
+        lookupTerm,
+        addEntity,
+        addFacet,
+        addAction,
+        addTopic,
+        buildIndex,
+        lemmatizeIndex,
+        expandWithRelatedTerms,
+        expandWithWordNet,
+        loadFromExport
+    };
+    // what is a term?
+    // - entity name
+    // - every type in entity
+    // - every facet in entity
+    // - every topic
+    // - every action verbs joined
+    // - action subject entity name
+    // - action object entity name
+    // - action inderect object entity name
+    // - action params
+    function getTerms() {
+        return [...map.keys()];
+    }
+    function lookupTerm(term) {
+        return map.get(term);
+    }
+    function removeTerm(term) {
+        map.delete(term);
+    }
+    function addTerm(term, semanticRefOrdinal) {
+        if (!term) {
+            return term;
+        }
+        term = term.toLowerCase();
+        const existing = map.get(term);
+        // Handle both SemanticRefOrdinal (number) and ScoredSemanticRefOrdinal (object with score)
+        let scoredOrdinal;
+        if (typeof semanticRefOrdinal === 'number') {
+            scoredOrdinal = {
+                semanticRefOrdinal,
+                score: 1.0
+            };
+        }
+        else {
+            scoredOrdinal = semanticRefOrdinal;
+        }
+        if (!existing) {
+            map.set(term, [scoredOrdinal]);
+        }
+        else {
+            map.set(term, [...existing, scoredOrdinal]);
+        }
+        return term;
+    }
+    function addEntity(entity, refOrdinal) {
+        // console.log("ADDING ENTITY", entity)
+        if (!entity.type) {
+            // TODO this is a hack we need to add some sort of validation
+            // to the fine tuning output so we don't get malformed objects
+            return;
+        }
+        addTerm(entity.name, refOrdinal);
+        entity.type.forEach((subType) => {
+            addTerm(subType, refOrdinal);
+        });
+        if (entity.facets) {
+            entity.facets.forEach(facet => {
+                addFacet(facet, refOrdinal);
+            });
+        }
+    }
+    function addFacet(facet, refOrdinal) {
+        addTerm(facet.name, refOrdinal);
+        if (facet.value) {
+            addTerm(facet.value.toString(), refOrdinal);
+        }
+    }
+    function addAction(action, refOrdinal) {
+        // console.log("ADDING ACTION", action)
+        const actionTerm = action.verbs.join(" ");
+        addTerm(actionTerm, refOrdinal);
+        addTerm(action.objectEntityName, refOrdinal);
+        addTerm(action.indirectObjectEntityName, refOrdinal);
+        addTerm(action.subjectEntityName, refOrdinal);
+        if (action.params) {
+            action.params.forEach((param) => {
+                if (typeof (param) === "string") {
+                    addTerm(param, refOrdinal);
+                }
+                else {
+                    addTerm(param.name, refOrdinal);
+                    // same here this is a crutch for malformed llm output
+                    if (param.value) {
+                        addTerm(param.value.toString(), refOrdinal);
+                    }
+                }
+            });
+        }
+    }
+    function addTopic(topic, refOrdinal) {
+        addTerm(topic.text, refOrdinal);
+    }
+    function buildIndex(semanticRefs) {
+        semanticRefs.getAll().forEach((ref) => {
+            switch (ref.knowledgeType) {
+                case "entity": {
+                    addEntity(ref.knowledge, ref.semanticRefOrdinal);
+                    break;
+                }
+                case "action": {
+                    addAction(ref.knowledge, ref.semanticRefOrdinal);
+                    break;
+                }
+                case "topic": {
+                    addTopic(ref.knowledge, ref.semanticRefOrdinal);
+                    break;
+                }
+                default: {
+                    break;
+                }
+            }
+        });
+    }
+    function loadFromExport(items) {
+        // Clear existing index
+        map.clear();
+        // Load all terms with their scored ordinals
+        for (const item of items) {
+            map.set(item.term.toLowerCase(), item.semanticRefOrdinals);
+        }
+    }
+    function lemmatizeIndex() {
+        const terms = getTerms();
+        const lemmaMap = new Map();
+        let lemmasAdded = 0;
+        for (const term of terms) {
+            // Skip very short terms or terms with spaces (phrases)
+            if (term.length < 3 || term.includes(" ")) {
+                continue;
+            }
+            // Get the semantic refs for this term
+            const refs = map.get(term);
+            if (!refs || refs.length === 0) {
+                continue;
+            }
+            // Try lemmatizing as different parts of speech
+            const nounLemma = lemmatizer.noun(term);
+            const verbLemma = lemmatizer.verb(term);
+            const adjLemma = lemmatizer.adjective(term);
+            // Collect unique lemmas that differ from the original
+            const lemmas = new Set();
+            if (nounLemma !== term)
+                lemmas.add(nounLemma);
+            if (verbLemma !== term)
+                lemmas.add(verbLemma);
+            if (adjLemma !== term)
+                lemmas.add(adjLemma);
+            // Add each lemma to the index pointing to the same refs
+            for (const lemma of lemmas) {
+                // Skip if lemma is already in the index with the same refs
+                const existingRefs = map.get(lemma);
+                if (!existingRefs) {
+                    // New lemma - add all refs
+                    map.set(lemma, [...refs]);
+                    lemmasAdded++;
+                    lemmaMap.set(term, lemma);
+                }
+                else {
+                    // Lemma exists - merge refs (avoid duplicates)
+                    const existingOrdinals = new Set(existingRefs.map(r => r.semanticRefOrdinal));
+                    const newRefs = refs.filter(r => !existingOrdinals.has(r.semanticRefOrdinal));
+                    if (newRefs.length > 0) {
+                        map.set(lemma, [...existingRefs, ...newRefs]);
+                    }
+                }
+            }
+        }
+        return {
+            termsProcessed: terms.length,
+            lemmasAdded,
+            lemmaMap
+        };
+    }
+    async function expandWithRelatedTerms(model, options) {
+        const { relatedTermWeight = 0.9, batchSizePercent = 0.1, maxBatchSize = 50, minBatchSize = 5, maxTerms } = options ?? {};
+        const terms = getTerms();
+        const expansionMap = new Map();
+        let relatedTermsAdded = 0;
+        // Filter terms: skip very short terms, phrases, and common stop words
+        const stopWords = new Set(['the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'dare', 'ought', 'used', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'between', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just', 'and', 'but', 'if', 'or', 'because', 'until', 'while', 'this', 'that', 'these', 'those', 'it', 'its']);
+        let eligibleTerms = terms.filter(term => term.length >= 3 &&
+            !term.includes(' ') &&
+            !stopWords.has(term));
+        // Limit total terms if maxTerms is specified
+        if (maxTerms !== undefined && maxTerms < eligibleTerms.length) {
+            eligibleTerms = eligibleTerms.slice(0, maxTerms);
+        }
+        if (eligibleTerms.length === 0) {
+            return { termsProcessed: 0, relatedTermsAdded: 0, expansionMap };
+        }
+        // Calculate batch size
+        const calculatedBatchSize = Math.floor(eligibleTerms.length * batchSizePercent);
+        const batchSize = Math.max(minBatchSize, Math.min(maxBatchSize, calculatedBatchSize));
+        // Create translator for related terms
+        const schema = loadSchema(["relatedTermsSchema.ts"], import.meta.url);
+        const validator = createTypeScriptJsonValidator(schema, "RelatedTermsResponse");
+        const translator = createJsonTranslator(model, validator);
+        translator.createRequestPrompt = (request) => {
+            return (`You are a service that finds semantically related words for search index expansion.\n` +
+                `Given a list of terms, return synonyms and semantically related words for each.\n\n` +
+                `IMPORTANT RULES:\n` +
+                `- Do NOT include morphological variants (plurals, verb tenses, etc.) - only semantically different words\n` +
+                `- Focus on words that users might use interchangeably when searching\n` +
+                `- Include synonyms, near-synonyms, and closely related concepts\n` +
+                `- Keep related terms to 2-5 per term (quality over quantity)\n` +
+                `- Skip proper nouns (names of people, places, brands)\n\n` +
+                `Examples:\n` +
+                `- book -> [novel, publication, text, volume]\n` +
+                `- buy -> [purchase, acquire, obtain]\n` +
+                `- happy -> [joyful, pleased, glad, content]\n` +
+                `- psychology -> [mental health, counseling, therapy]\n\n` +
+                `The following TypeScript definitions describe the expected response format:\n` +
+                `\`\`\`\n${schema}\`\`\`\n` +
+                `Input terms:\n${request}\n\n` +
+                `Return a JSON object of type RelatedTermsResponse with related terms for each input term:\n`);
+        };
+        // Process in batches
+        for (let i = 0; i < eligibleTerms.length; i += batchSize) {
+            const batch = eligibleTerms.slice(i, i + batchSize);
+            console.log(`Expanding terms batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(eligibleTerms.length / batchSize)} (${batch.length} terms)`);
+            try {
+                const request = batch.join(', ');
+                const result = await translator.translate(request);
+                if (!result.success) {
+                    console.warn(`Failed to expand batch: ${result.message}`);
+                    continue;
+                }
+                // Process the response
+                for (const termWithRelated of result.data.terms) {
+                    const originalTerm = termWithRelated.term.toLowerCase();
+                    const refs = map.get(originalTerm);
+                    if (!refs || refs.length === 0) {
+                        continue;
+                    }
+                    const addedRelated = [];
+                    for (const relatedTerm of termWithRelated.relatedTerms) {
+                        const normalizedRelated = relatedTerm.toLowerCase();
+                        // Skip if it's the same as the original
+                        if (normalizedRelated === originalTerm) {
+                            continue;
+                        }
+                        // Add the related term pointing to the same refs with lower weight
+                        for (const ref of refs) {
+                            const weightedRef = {
+                                semanticRefOrdinal: ref.semanticRefOrdinal,
+                                score: ref.score * relatedTermWeight
+                            };
+                            // Check if this term already exists with this ref
+                            const existingRefs = map.get(normalizedRelated);
+                            if (existingRefs) {
+                                const alreadyHasRef = existingRefs.some(r => r.semanticRefOrdinal === ref.semanticRefOrdinal);
+                                if (!alreadyHasRef) {
+                                    map.set(normalizedRelated, [...existingRefs, weightedRef]);
+                                }
+                            }
+                            else {
+                                map.set(normalizedRelated, [weightedRef]);
+                            }
+                        }
+                        addedRelated.push(normalizedRelated);
+                        relatedTermsAdded++;
+                    }
+                    if (addedRelated.length > 0) {
+                        expansionMap.set(originalTerm, addedRelated);
+                    }
+                }
+            }
+            catch (error) {
+                console.error(`Error expanding batch:`, error);
+            }
+        }
+        return {
+            termsProcessed: eligibleTerms.length,
+            relatedTermsAdded,
+            expansionMap
+        };
+    }
+    async function expandWithWordNet(options) {
+        const { relatedTermWeight = 0.9, maxTerms } = options ?? {};
+        const wordnet = new natural.WordNet();
+        const terms = getTerms();
+        const expansionMap = new Map();
+        let relatedTermsAdded = 0;
+        // Filter terms: skip very short terms, phrases, and common stop words
+        const stopWords = new Set(['the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'dare', 'ought', 'used', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'between', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just', 'and', 'but', 'if', 'or', 'because', 'until', 'while', 'this', 'that', 'these', 'those', 'it', 'its']);
+        let eligibleTerms = terms.filter(term => term.length >= 3 &&
+            !term.includes(' ') &&
+            !stopWords.has(term));
+        // Limit total terms if maxTerms is specified
+        if (maxTerms !== undefined && maxTerms < eligibleTerms.length) {
+            eligibleTerms = eligibleTerms.slice(0, maxTerms);
+        }
+        if (eligibleTerms.length === 0) {
+            return { termsProcessed: 0, relatedTermsAdded: 0, expansionMap };
+        }
+        // Helper to get synonyms from WordNet (promisified)
+        const getSynonyms = (word) => {
+            return new Promise((resolve) => {
+                const synonyms = new Set();
+                wordnet.lookup(word, (results) => {
+                    if (results && results.length > 0) {
+                        for (const result of results) {
+                            // Add synonyms from the synset
+                            if (result.synonyms) {
+                                for (const syn of result.synonyms) {
+                                    const normalized = syn.toLowerCase().replace(/_/g, ' ');
+                                    // Skip multi-word phrases and the original word
+                                    if (!normalized.includes(' ') && normalized !== word) {
+                                        synonyms.add(normalized);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    resolve([...synonyms].slice(0, 5)); // Limit to 5 synonyms per term
+                });
+            });
+        };
+        console.log(`Expanding ${eligibleTerms.length} terms using WordNet...`);
+        let processed = 0;
+        for (const term of eligibleTerms) {
+            const refs = map.get(term);
+            if (!refs || refs.length === 0) {
+                continue;
+            }
+            const synonyms = await getSynonyms(term);
+            const addedRelated = [];
+            for (const synonym of synonyms) {
+                // Add the related term pointing to the same refs with lower weight
+                for (const ref of refs) {
+                    const weightedRef = {
+                        semanticRefOrdinal: ref.semanticRefOrdinal,
+                        score: ref.score * relatedTermWeight
+                    };
+                    // Check if this term already exists with this ref
+                    const existingRefs = map.get(synonym);
+                    if (existingRefs) {
+                        const alreadyHasRef = existingRefs.some(r => r.semanticRefOrdinal === ref.semanticRefOrdinal);
+                        if (!alreadyHasRef) {
+                            map.set(synonym, [...existingRefs, weightedRef]);
+                        }
+                    }
+                    else {
+                        map.set(synonym, [weightedRef]);
+                    }
+                }
+                addedRelated.push(synonym);
+                relatedTermsAdded++;
+            }
+            if (addedRelated.length > 0) {
+                expansionMap.set(term, addedRelated);
+            }
+            processed++;
+            if (processed % 100 === 0) {
+                console.log(`  Processed ${processed}/${eligibleTerms.length} terms...`);
+            }
+        }
+        console.log(`WordNet expansion complete: ${relatedTermsAdded} related terms added`);
+        return {
+            termsProcessed: eligibleTerms.length,
+            relatedTermsAdded,
+            expansionMap
+        };
+    }
+}
+//# sourceMappingURL=semanticRefIndex.js.map

package/dist/knowpro/semanticRefIndex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semanticRefIndex.js","sourceRoot":"","sources":["../../src/knowpro/semanticRefIndex.ts"],"names":[],"mappings":"AAEA,OAAO,UAAU,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,OAAO,MAAM,SAAS,CAAC;AA+C9B,MAAM,UAAU,8BAA8B;IAE1C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAsC,CAAC;IAE1D,OAAO;QACH,QAAQ;QACR,OAAO;QACP,UAAU;QACV,UAAU;QACV,SAAS;QACT,QAAQ;QACR,SAAS;QACT,QAAQ;QACR,UAAU;QACV,cAAc;QACd,sBAAsB;QACtB,iBAAiB;QACjB,cAAc;KACjB,CAAA;IAED,kBAAkB;IAClB,gBAAgB;IAChB,yBAAyB;IACzB,0BAA0B;IAC1B,gBAAgB;IAChB,8BAA8B;IAC9B,+BAA+B;IAC/B,8BAA8B;IAC9B,uCAAuC;IACvC,kBAAkB;IAElB,SAAS,QAAQ;QACb,OAAO,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED,SAAS,UAAU,CAAC,IAAY;QAC5B,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,SAAS,UAAU,CAAC,IAAY;QAC5B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IAED,SAAS,OAAO,CAAC,IAAY,EAAE,kBAAiE;QAC5F,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE/B,2FAA2F;QAC3F,IAAI,aAAuC,CAAC;QAC5C,IAAI,OAAO,kBAAkB,KAAK,QAAQ,EAAE,CAAC;YACzC,aAAa,GAAG;gBACZ,kBAAkB;gBAClB,KAAK,EAAE,GAAG;aACb,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,aAAa,GAAG,kBAAkB,CAAC;QACvC,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACJ,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,SAAS,SAAS,CAAC,MAAsB,EAAE,UAA8B;QACrE,uCAAuC;QACvC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACf,6DAA6D;YAC7D,8DAA8D;YAC9D,OAAO;QACX,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC5B,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACjC,CAAC,CAAC,CAAA;QAEF,IAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACf,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC1B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,SAAS,QAAQ,CAAC,KAAY,EAAE,UAA8B;QAC1D,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;IACL,CAAC;IAED,SAAS,SAAS,CAAC,MAAc,EAAE,UAA8B;QAC7D,uCAAuC;QACvC,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAChC,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;QAC7C,OAAO,CAAC,MAAM,CAAC,wBAAwB,EAAE,UAAU,CAAC,CAAC;QACrD,OAAO,CAAC,MAAM,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;QAE9C,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC5B,IAAI,OAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;gBAC/B,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;oBAEhC,sDAAsD;oBACtD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;wBACd,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,UAAU,CAAC,CAAC;oBAChD,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAA;QACN,CAAC;IACL,CAAC;IAED,SAAS,QAAQ,CAAC,KAAY,EAAE,UAA8B;QAC1D,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACpC,CAAC;IAED,SAAS,UAAU,CAAC,YAAoC;QACpD,YAAY,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,GAAgB,EAAE,EAAE;YAC/C,QAAO,GAAG,CAAC,aAAa,EAAE,CAAC;gBACvB,KAAK,QAAQ,CAAC,CAAC,CAAC;oBACZ,SAAS,CAAC,GAAG,CAAC,SAA2B,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC;oBACnE,MAAM;gBACV,CAAC;gBACD,KAAK,QAAQ,CAAC,CAAC,CAAC;oBACZ,SAAS,CAAC,GAAG,CAAC,SAAmB,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC;oBAC3D,MAAM;gBACV,CAAC;gBACD,KAAK,OAAQ,CAAC,CAAC,CAAC;oBACZ,QAAQ,CAAC,GAAG,CAAC,SAAkB,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC;oBACzD,MAAM;gBACV,CAAC;gBACD,OAAO,CAAC,CAAC,CAAC;oBACN,MAAM;gBACV,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,SAAS,cAAc,CAAC,KAA+E;QACnG,uBAAuB;QACvB,GAAG,CAAC,KAAK,EAAE,CAAC;QAEZ,4CAA4C;QAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;IAED,SAAS,cAAc;QACnB,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,uDAAuD;YACvD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxC,SAAS;YACb,CAAC;YAED,sCAAsC;YACtC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,SAAS;YACb,CAAC;YAED,+CAA+C;YAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAE5C,sDAAsD;YACtD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;YACjC,IAAI,SAAS,KAAK,IAAI;gBAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,SAAS,KAAK,IAAI;gBAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,QAAQ,KAAK,IAAI;gBAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAE5C,wDAAwD;YACxD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,2DAA2D;gBAC3D,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAEpC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,2BAA2B;oBAC3B,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;oBAC1B,WAAW,EAAE,CAAC;oBACd,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC9B,CAAC;qBAAM,CAAC;oBACJ,+CAA+C;oBAC/C,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;oBAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;oBAC9E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACrB,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;oBAClD,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO;YACH,cAAc,EAAE,KAAK,CAAC,MAAM;YAC5B,WAAW;YACX,QAAQ;SACX,CAAC;IACN,CAAC;IAED,KAAK,UAAU,sBAAsB,CACjC,KAA4B,EAC5B,OAAsC;QAEtC,MAAM,EACF,iBAAiB,GAAG,GAAG,EACvB,gBAAgB,GAAG,GAAG,EACtB,YAAY,GAAG,EAAE,EACjB,YAAY,GAAG,CAAC,EAChB,QAAQ,EACX,GAAG,OAAO,IAAI,EAAE,CAAC;QAElB,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;QACjD,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,sEAAsE;QACtE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAEztB,IAAI,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CACpC,IAAI,CAAC,MAAM,IAAI,CAAC;YAChB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACnB,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CACvB,CAAC;QAEF,6CAA6C;QAC7C,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;YAC5D,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,EAAE,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC;QACrE,CAAC;QAED,uBAAuB;QACvB,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,GAAG,gBAAgB,CAAC,CAAC;QAChF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAEtF,sCAAsC;QACtC,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,6BAA6B,CAAuB,MAAM,EAAE,sBAAsB,CAAC,CAAC;QACtG,MAAM,UAAU,GAAG,oBAAoB,CAAuB,KAAK,EAAE,SAAS,CAAC,CAAC;QAEhF,UAAU,CAAC,mBAAmB,GAAG,CAAC,OAAe,EAAE,EAAE;YACjD,OAAO,CACH,uFAAuF;gBACvF,qFAAqF;gBACrF,oBAAoB;gBACpB,4GAA4G;gBAC5G,wEAAwE;gBACxE,mEAAmE;gBACnE,gEAAgE;gBAChE,2DAA2D;gBAC3D,aAAa;gBACb,gDAAgD;gBAChD,wCAAwC;gBACxC,+CAA+C;gBAC/C,0DAA0D;gBAC1D,+EAA+E;gBAC/E,WAAW,MAAM,UAAU;gBAC3B,iBAAiB,OAAO,MAAM;gBAC9B,6FAA6F,CAChG,CAAC;QACN,CAAC,CAAC;QAEF,qBAAqB;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,SAAS,CAAC,KAAK,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;YAE7I,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBAEnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAClB,OAAO,CAAC,IAAI,CAAC,2BAA2B,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;oBAC1D,SAAS;gBACb,CAAC;gBAED,uBAAuB;gBACvB,KAAK,MAAM,eAAe,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC9C,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;oBACxD,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;oBAEnC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC7B,SAAS;oBACb,CAAC;oBAED,MAAM,YAAY,GAAa,EAAE,CAAC;oBAElC,KAAK,MAAM,WAAW,IAAI,eAAe,CAAC,YAAY,EAAE,CAAC;wBACrD,MAAM,iBAAiB,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;wBAEpD,wCAAwC;wBACxC,IAAI,iBAAiB,KAAK,YAAY,EAAE,CAAC;4BACrC,SAAS;wBACb,CAAC;wBAED,mEAAmE;wBACnE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;4BACrB,MAAM,WAAW,GAA6B;gCAC1C,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gCAC1C,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,iBAAiB;6BACvC,CAAC;4BAEF,kDAAkD;4BAClD,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;4BAChD,IAAI,YAAY,EAAE,CAAC;gCACf,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB,KAAK,GAAG,CAAC,kBAAkB,CAAC,CAAC;gCAC9F,IAAI,CAAC,aAAa,EAAE,CAAC;oCACjB,GAAG,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,GAAG,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;gCAC/D,CAAC;4BACL,CAAC;iCAAM,CAAC;gCACJ,GAAG,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;4BAC9C,CAAC;wBACL,CAAC;wBAED,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;wBACrC,iBAAiB,EAAE,CAAC;oBACxB,CAAC;oBAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1B,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;oBACjD,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;YACnD,CAAC;QACL,CAAC;QAED,OAAO;YACH,cAAc,EAAE,aAAa,CAAC,MAAM;YACpC,iBAAiB;YACjB,YAAY;SACf,CAAC;IACN,CAAC;IAED,KAAK,UAAU,iBAAiB,CAC5B,OAAiC;QAEjC,MAAM,EACF,iBAAiB,GAAG,GAAG,EACvB,QAAQ,EACX,GAAG,OAAO,IAAI,EAAE,CAAC;QAElB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;QACjD,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,sEAAsE;QACtE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QAEztB,IAAI,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CACpC,IAAI,CAAC,MAAM,IAAI,CAAC;YAChB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YACnB,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CACvB,CAAC;QAEF,6CAA6C;QAC7C,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC;YAC5D,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACrD,CAAC;QAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,EAAE,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC;QACrE,CAAC;QAED,oDAAoD;QACpD,MAAM,WAAW,GAAG,CAAC,IAAY,EAAqB,EAAE;YACpD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC3B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;gBACnC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE;oBAC7B,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;4BAC3B,+BAA+B;4BAC/B,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gCAClB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;oCAChC,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oCACxD,gDAAgD;oCAChD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;wCACnD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;oCAC7B,CAAC;gCACL,CAAC;4BACL,CAAC;wBACL,CAAC;oBACL,CAAC;oBACD,OAAO,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,+BAA+B;gBACvE,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;QACP,CAAC,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,aAAa,aAAa,CAAC,MAAM,yBAAyB,CAAC,CAAC;QACxE,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,SAAS;YACb,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,YAAY,GAAa,EAAE,CAAC;YAElC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,mEAAmE;gBACnE,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACrB,MAAM,WAAW,GAA6B;wBAC1C,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;wBAC1C,KAAK,EAAE,GAAG,CAAC,KAAK,GAAG,iBAAiB;qBACvC,CAAC;oBAEF,kDAAkD;oBAClD,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtC,IAAI,YAAY,EAAE,CAAC;wBACf,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB,KAAK,GAAG,CAAC,kBAAkB,CAAC,CAAC;wBAC9F,IAAI,CAAC,aAAa,EAAE,CAAC;4BACjB,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;wBACrD,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACJ,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;oBACpC,CAAC;gBACL,CAAC;gBAED,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,iBAAiB,EAAE,CAAC;YACxB,CAAC;YAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACzC,CAAC;YAED,SAAS,EAAE,CAAC;YACZ,IAAI,SAAS,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,GAAG,CAAC,eAAe,SAAS,IAAI,aAAa,CAAC,MAAM,WAAW,CAAC,CAAC;YAC7E,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,+BAA+B,iBAAiB,sBAAsB,CAAC,CAAC;QAEpF,OAAO;YACH,cAAc,EAAE,aAAa,CAAC,MAAM;YACpC,iBAAiB;YACjB,YAAY;SACf,CAAC;IACN,CAAC;AAEL,CAAC"}

package/dist/knowpro/storageProvider.d.ts

@@ -0,0 +1,7 @@
+import type { IMessageCollection, ISemanticRefCollection, IStorageProvider } from "./interfaces.js";
+import { type Message } from "./conversation.js";
+export interface InMemoryStorageProvider extends IStorageProvider {
+    loadFromFile: (filePath: string, semanticRefs: ISemanticRefCollection, messages: IMessageCollection<Message>) => void;
+}
+export declare function createInMemoryStorageProvider(): InMemoryStorageProvider;
+//# sourceMappingURL=storageProvider.d.ts.map

package/dist/knowpro/storageProvider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storageProvider.d.ts","sourceRoot":"","sources":["../../src/knowpro/storageProvider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGpG,OAAO,EAAiB,KAAK,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAGhE,MAAM,WAAW,uBAAwB,SAAQ,gBAAgB;IAC7D,YAAY,EAAE,CACV,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,sBAAsB,EACpC,QAAQ,EAAE,kBAAkB,CAAC,OAAO,CAAC,KACpC,IAAI,CAAC;CACb;AAID,wBAAgB,6BAA6B,IAAI,uBAAuB,CAuEvE"}

package/dist/knowpro/storageProvider.js

@@ -0,0 +1,61 @@
+import { createMessageCollection, createSemanticRefCollection } from "./collections.js";
+import { readFileSync } from "node:fs";
+import { createMessage } from "./conversation.js";
+import { populateSemanticRef } from "./semanticRef.js";
+const SEPERATOR = "================================================================================";
+export function createInMemoryStorageProvider() {
+    function loadFromFile(filePath, semanticRefs, messages) {
+        const knowledgeFile = readFileSync(filePath, "utf-8");
+        const sections = knowledgeFile.split(SEPERATOR);
+        let lossCount = 0;
+        sections.forEach((section) => {
+            // if (semanticRefOrdinal != semanticRefs.length) {
+            //     console.log("ERROR", semanticRefOrdinal, semanticRefs.length);
+            //     throw Error("HELP SOMETHING WENT WRONG THESE MUST BE THE SAME LENGTH");
+            // }
+            const jsonStart = section.indexOf("{");
+            // Get and add message
+            const content = section.substring(0, jsonStart);
+            const message = createMessage(content, []);
+            // filter our the empty chunks
+            if (content.length < 10) {
+                return;
+            }
+            // 1. merge entities and actions
+            // 2. make a token limit
+            // 3. decide how to preseent merged e & a
+            //    into the answer propmt densely
+            // what did julia do in the forest?
+            // how did julia meet winston?
+            // what color is julia's sash?
+            // what is the book winston gets from obrien about?
+            // what is the uniform of the inner party?
+            // Get knowledge and add semantic refs
+            const jsonString = section.substring(jsonStart).split("\n")[0];
+            try {
+                const knowledgeResponse = JSON.parse(jsonString);
+                if (knowledgeResponse) {
+                    semanticRefs = populateSemanticRef(semanticRefs, knowledgeResponse, semanticRefs.length, messages.length);
+                    // semanticRefOrdinal += ordinalModifier;
+                    messages.append(message);
+                }
+            }
+            catch {
+                lossCount += 1;
+            }
+        });
+        console.log("LOSS COUNT: ", lossCount, lossCount / sections.length, sections.length);
+        console.log(messages.length, semanticRefs.length);
+    }
+    function close() {
+        console.log("Closing storage provider.");
+        return;
+    }
+    return {
+        createMessageCollection,
+        createSemanticRefCollection,
+        loadFromFile,
+        close
+    };
+}
+//# sourceMappingURL=storageProvider.js.map

package/dist/knowpro/storageProvider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storageProvider.js","sourceRoot":"","sources":["../../src/knowpro/storageProvider.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAC;AACxF,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAgB,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAUvD,MAAM,SAAS,GAAG,kFAAkF,CAAC;AAErG,MAAM,UAAU,6BAA6B;IAEzC,SAAS,YAAY,CACjB,QAAgB,EAChB,YAAoC,EACpC,QAAqC;QAErC,MAAM,aAAa,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACzB,mDAAmD;YACnD,qEAAqE;YACrE,8EAA8E;YAC9E,IAAI;YACJ,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAEvC,sBAAsB;YACtB,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YAChD,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAE3C,8BAA8B;YAC9B,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACtB,OAAO;YACX,CAAC;YAED,gCAAgC;YAChC,wBAAwB;YACxB,yCAAyC;YACzC,oCAAoC;YAEpC,mCAAmC;YACnC,8BAA8B;YAC9B,8BAA8B;YAC9B,mDAAmD;YACnD,0CAA0C;YAE1C,sCAAsC;YACtC,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,IAAI,CAAC;gBACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBACjD,IAAI,iBAAiB,EAAE,CAAC;oBACpB,YAAY,GAAG,mBAAmB,CAC9B,YAAY,EACZ,iBAAiB,EACjB,YAAY,CAAC,MAAM,EACnB,QAAQ,CAAC,MAAM,CAClB,CAAC;oBACF,yCAAyC;oBACzC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC7B,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS,IAAI,CAAC,CAAC;YACnB,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,EAAE,SAAS,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrF,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC,CAAA;IACrD,CAAC;IAED,SAAS,KAAK;QACV,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,OAAO;IACX,CAAC;IAED,OAAO;QACH,uBAAuB;QACvB,2BAA2B;QAC3B,YAAY;QACZ,KAAK;KACR,CAAA;AACL,CAAC"}