ai-compare-candidates 0.0.2 → 0.0.4

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","names":["matches:string[]","match:RegExpExecArray|null","selectedCandidates:Candidate[]","summaries:string[]","summarisableSubstringIndices:AICompareCandidates.SummarisableSubstringIndices"],"sources":["../src/index.ts"],"sourcesContent":["import{\n\tenv,\n\tpipeline,\n\tAutoTokenizer,\n\tTextGenerationPipeline,\n\tProgressInfo,\n\tProgressCallback,\n\tSummarizationPipeline,\n\tFeatureExtractionPipeline,\n\tPreTrainedTokenizer,\n\tTextGenerationConfig\n}from '@huggingface/transformers';\nimport {MemoryVectorStore} from '@langchain/classic/vectorstores/memory';\nimport {Embeddings} from '@langchain/core/embeddings';\nimport lodash from 'lodash';\nimport jsan from 'jsan';\n\nexport class AICompareCandidates extends Embeddings{\n\treadonly env=env;\n\tDEBUG=true;\n\n\tgenerator:TextGenerationPipeline|null=null;\n\tgeneratorModelName='Xenova/LaMini-GPT-774M';\n\tgeneratorPromise:Promise<TextGenerationPipeline>|null=null;\n\tgeneratorProgressInfo:ProgressInfo=<ProgressInfo>{};\n\tgeneratorProgressCallback:ProgressCallback|null=null;\n\n\tsummariser:SummarizationPipeline|null=null;\n\tsummariserModelName='Xenova/distilbart-cnn-12-6';\n\tsummariserPromise:Promise<SummarizationPipeline>|null=null;\n\tsummariserProgressInfo:ProgressInfo=<ProgressInfo>{};\n\tsummariserProgressCallback:ProgressCallback|null=null;\n\n\tembedder:FeatureExtractionPipeline|null=null;\n\tembedderModelName='Xenova/all-MiniLM-L12-v2';\n\tembedderPromise:Promise<FeatureExtractionPipeline>|null=null;\n\tembedderProgressInfo:ProgressInfo=<ProgressInfo>{};\n\tembedderProgressCallback:ProgressCallback|null=null;\n\n\ttokeniser:PreTrainedTokenizer|null=null;\n\ttokeniserModelName=this.generatorModelName;\n\ttokeniserPromise:Promise<PreTrainedTokenizer>|null=null;\n\ttokeniserProgressInfo:ProgressInfo=<ProgressInfo>{};\n\ttokeniserProgressCallback:ProgressCallback|null=null;\n\n\tgenerateSearchAreasMaxNewTokens=64;\n\tgenerateSearchAreasTemperature=0.35;\n\tgenerateSearchAreasRepetitionPenalty=1.5;\n\n\trankingMaxNewTokens=64;\n\trankingTemperature=0.35;\n\trankingRepetitionPenalty=1.5;\n\n\ttargetSummarisedStringTokenCount=420;\n\n\tstatic{\n\t\tenv.localModelPath='';\n\t\tenv.allowRemoteModels=true;\n\t\tenv.allowLocalModels=false;\n\t}\n\n\tconstructor(){\n\t\tsuper({});\n\t}\n\t\n\tasync loadGenerator({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(typeof modelName==='string'&&modelName)this.generatorModelName=modelName;\n\t\tif(!this.generatorModelName)throw new Error('Invalid generator model name');\n\t\tif(progressCallback)this.generatorProgressCallback=progressCallback;\n\t\t//ts-ignore is needed due to frequent error TS2590: Expression produces a union type that is too complex to represent.\n\t\t//@ts-ignore\n\t\tthis.generatorPromise=pipeline('text-generation',this.generatorModelName,{\n\t\t\tdevice:'webgpu',\n\t\t\tprogress_callback:progressInfo=>{\n\t\t\t\tif(this.DEBUG)console.log(jsan.stringify(progressInfo));\n\t\t\t\tObject.assign(this.generatorProgressInfo,progressInfo);\n\t\t\t\treturn this.generatorProgressCallback?.(progressInfo);\n\t\t\t}\n\t\t});\n\t\tthis.generator=await this.generatorPromise;\n\t\treturn this.generator;\n\t}\n\n\tasync checkGeneratorLoaded(){\n\t\tif(!this.generatorPromise)this.loadGenerator();\n\t\tif(!this.generator)await this.generatorPromise;\n\t\tif(!this.generator)throw new Error('Unable to load generator');\n\t}\n\n\tasync loadSummariser({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(typeof modelName==='string'&&modelName)this.summariserModelName=modelName;\n\t\tif(!this.summariserModelName)throw new Error('Invalid summariser model name');\n\t\tif(progressCallback)this.summariserProgressCallback=progressCallback;\n\t\t//ts-ignore is needed due to frequent error TS2590: Expression produces a union type that is too complex to represent.\n\t\t//@ts-ignore\n\t\tthis.summariserPromise=pipeline('summarization',this.summariserModelName,{\n\t\t\tdevice:'webgpu',\n\t\t\tprogress_callback:progressInfo=>{\n\t\t\t\tif(this.DEBUG)console.log(jsan.stringify(progressInfo));\n\t\t\t\tObject.assign(this.summariserProgressInfo,progressInfo);\n\t\t\t\treturn this.summariserProgressCallback?.(progressInfo);\n\t\t\t}\n\t\t});\n\t\tthis.summariser=await this.summariserPromise;\n\t\treturn this.summariser;\n\t}\n\n\tasync checkSummariserLoaded(){\n\t\tif(!this.summariserPromise)this.loadSummariser();\n\t\tif(!this.summariser)await this.summariserPromise;\n\t\tif(!this.summariser)throw new Error('Unable to load summariser');\n\t}\n\n\tasync loadEmbedder({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(typeof modelName==='string'&&modelName)this.embedderModelName=modelName;\n\t\tif(!this.embedderModelName)throw new Error('Invalid embedder model name');\n\t\tif(progressCallback)this.embedderProgressCallback=progressCallback;\n\t\t//ts-ignore is needed due to frequent error TS2590: Expression produces a union type that is too complex to represent.\n\t\t//@ts-ignore\n\t\tthis.embedderPromise=pipeline('feature-extraction',this.embedderModelName,{\n\t\t\tdevice:'webgpu',\n\t\t\tprogress_callback:progressInfo=>{\n\t\t\t\tif(this.DEBUG)console.log(jsan.stringify(progressInfo));\n\t\t\t\tObject.assign(this.embedderProgressInfo,progressInfo);\n\t\t\t\treturn this.embedderProgressCallback?.(progressInfo);\n\t\t\t}\n\t\t});\n\t\tthis.embedder=await this.embedderPromise;\n\t\treturn this.embedder;\n\t}\n\n\tasync checkEmbedderLoaded(){\n\t\tif(!this.embedderPromise)this.loadEmbedder();\n\t\tif(!this.embedder)await this.embedderPromise;\n\t\tif(!this.embedder)throw new Error('Unable to load embedder');\n\t}\n\n\tasync loadTokeniser({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(typeof modelName==='string'&&modelName)this.tokeniserModelName=modelName;\n\t\tif(!this.tokeniserModelName)throw new Error('Invalid tokeniser model name');\n\t\tif(progressCallback)this.tokeniserProgressCallback=progressCallback;\n\t\t//ts-ignore is needed due to frequent error TS2590: Expression produces a union type that is too complex to represent.\n\t\t//@ts-ignore\n\t\tthis.tokeniserPromise=AutoTokenizer.from_pretrained(this.tokeniserModelName,{\n\t\t\tprogress_callback:progressInfo=>{\n\t\t\t\tif(this.DEBUG)console.log(jsan.stringify(progressInfo));\n\t\t\t\tObject.assign(this.tokeniserProgressInfo,progressInfo);\n\t\t\t\treturn this.tokeniserProgressCallback?.(progressInfo);\n\t\t\t}\n\t\t})\n\t\tthis.tokeniser=await this.tokeniserPromise;\n\t\treturn this.tokeniser;\n\t}\n\n\tasync checkTokeniserLoaded(){\n\t\tif(!this.tokeniserPromise)this.loadTokeniser();\n\t\tif(!this.tokeniser)await this.tokeniserPromise;\n\t\tif(!this.tokeniser)throw new Error('Unable to load tokeniser');\n\t}\n\n\tasync embedQuery(text:string):Promise<number[]>{\n\t\tawait this.checkEmbedderLoaded();\n\t\treturn Array.from((await this.embedder?.(text,{\n\t\t\tpooling:'mean',\n\t\t\tnormalize:true\n\t\t}))?.data);\n\t}\n\n\tasync embedDocuments(texts:string[]):Promise<number[][]>{\n\t\treturn Promise.all(texts.map(text=>this.embedQuery(text)));\n\t}\n\n\tgeneratePromptTemplate(prompt:string){\n\t\treturn 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n'+\n\t\t\t'### Instruction:\\n'+\n\t\t\tprompt+\n\t\t\t'\\n\\n### Response:';\n\t}\n\n\tdefaultGenerateSearchAreasInstruction(problemDescription:string){\n\t\treturn 'List the relevant subject areas for the following issues. Limit your response to 100 words.\\nIssues: \"'+problemDescription+'\"';\n\t}\n\n\tdefaultConvertCandidateToDocument<Candidate>({\n\t\tcandidate,\n\t\tindex\n\t}:AICompareCandidates.ConvertCandidateToDocumentArguments<Candidate>=<AICompareCandidates.ConvertCandidateToDocumentArguments<Candidate>>{}){\n\t\tlet document='Start of Candidate #'+index;\n\t\tfor(let i in candidate)document+='\\n'+lodash.startCase(i)+': '+(typeof candidate[i]==='object'?jsan.stringify(candidate[i]):String(candidate[i]));\n\t\tdocument+='\\nEnd of Candidate #'+index;\n\t\treturn document;\n\t}\n\n\tdefaultGenerateRankingInstruction({\n\t\tproblemDescription,\n\t\tsummaries,\n\t\tcandidatesForFinalSelection,\n\t\tcandidateIdentifierField\n\t}:AICompareCandidates.GenerateRankingInstructionArguments=<AICompareCandidates.GenerateRankingInstructionArguments>{}){\n\t\treturn 'Strictly follow these rules:\\n'+\n\t\t\t'1. Rank ONLY the top '+candidatesForFinalSelection+' candidates with one 15-word sentence explaining why\\n'+\n\t\t\t'2. Rank the candidates based on \"'+problemDescription.replace(/(\\r|\\n)/g,' ')+'\"\\n'+\n\t\t\t'3. If unclear, say \"Insufficient information to determine\"\\n\\n'+\n\t\t\t'Candidates:\\n\\n'+summaries.join('\\n\\n')+'\\n\\n'+\n\t\t\t'Format exactly:\\n'+\n\t\t\t'#1. \"Full '+lodash.startCase(candidateIdentifierField)+'\": 15-word explanation\\n'+\n\t\t\t'#2. ...';\n\t}\n\n\tregexIndexOf(text:string,regex:RegExp,startIndex:number){\n    \tlet indexInSuffix=text.slice(startIndex).search(regex);\n    \treturn indexInSuffix<0?indexInSuffix:indexInSuffix+startIndex;\n\t}\n\n\tdefaultExtractIdentifierFromCandidateDocument({\n\t\tcandidateDocument,\n\t\tcandidateIdentifierField\n\t}:AICompareCandidates.ExtractIdentifierFromCandidateDocumentArguments=<AICompareCandidates.ExtractIdentifierFromCandidateDocumentArguments>{}){\n\t\tif(this.DEBUG)console.log(candidateDocument,candidateIdentifierField);\n\t\tlet startCase=lodash.startCase(candidateIdentifierField);\n\t\tlet startIndex=candidateDocument.indexOf(startCase);\n\t\tif(startIndex<0)startIndex=candidateDocument.toLowerCase().indexOf(startCase.toLowerCase());\n\t\tif(this.DEBUG)console.log(startIndex);\n\t\tif(startIndex>=0)startIndex+=startCase.length;\n\t\tif(startIndex<0){\n\t\t\tstartIndex=candidateDocument.toLowerCase().indexOf(candidateIdentifierField.toLowerCase());\n\t\t\tif(startIndex>=0)startIndex+=candidateIdentifierField.length;\n\t\t}\n\t\tif(this.DEBUG)console.log(startIndex);\n\t\telse return '';\n\t\tstartIndex=candidateDocument.indexOf(':',startIndex);\n\t\tif(this.DEBUG)console.log(startIndex);\n\t\tif(startIndex<0)startIndex=this.regexIndexOf(candidateDocument,/\\s+/,startIndex);\n\t\tif(this.DEBUG)console.log(startIndex);\n\t\tif(startIndex<0)return '';\n\t\tlet endIndex=candidateDocument.indexOf('\\n',startIndex);\n\t\tif(endIndex<0)endIndex=candidateDocument.length;\n\t\tif(this.DEBUG)console.log(endIndex);\n\t\treturn candidateDocument.substring(startIndex,endIndex).trim();\n\t}\n\n\tdefaultExtractIdentifiersFromRationale(rationale:string){\n\t\tlet regex=/^\\s*#\\s*\\d+\\s*\\.?\\s*\"([^\"]+)\"/gm;\n\t\tlet matches:string[]=[];\n\t\tfor(let match:RegExpExecArray|null;Array.isArray(match=regex.exec(rationale));)if(match[1])matches.push(match[1]);\n\t\treturn matches;\n\t}\n\n\tasync compareCandidates<Candidate>({\n\t\tcandidates,\n\t\tproblemDescription='',\n\t\tgenerateSearchAreasInstruction=this.defaultGenerateSearchAreasInstruction.bind(this),\n\t\tconvertCandidateToDocument=this.defaultConvertCandidateToDocument.bind(this),\n\t\tcandidatesForInitialSelection=2,\n\t\tcandidatesForFinalSelection=1,\n\t\tgenerateRankingInstruction=this.defaultGenerateRankingInstruction.bind(this),\n\t\textractIdentifiersFromRationale=this.defaultExtractIdentifiersFromRationale.bind(this),\n\t\textractIdentifierFromCandidateDocument=this.defaultExtractIdentifierFromCandidateDocument.bind(this),\n\t\tcandidateIdentifierField=undefined,\n\t\tgetSummarisableSubstringIndices\n\t}:AICompareCandidates.CompareArguments<Candidate>=<AICompareCandidates.CompareArguments<Candidate>>{}):Promise<AICompareCandidates.CompareCandidatesReturn<Candidate>|void>{\n\t\tif(!Array.isArray(candidates)||candidates.length<=0)throw new Error('No candidates provided');\n\t\tcandidatesForInitialSelection=lodash.toSafeInteger(candidatesForInitialSelection);\n\t\tif(candidatesForInitialSelection<=0)throw new Error('Candidates for initial selection must be a positive integer bigger than 0');\n\t\tcandidatesForFinalSelection=lodash.toSafeInteger(candidatesForFinalSelection);\n\t\tif(candidatesForFinalSelection<=0)throw new Error('Candidates for initial selection must be a positive integer bigger than 0');\n\t\tif(candidatesForInitialSelection<candidatesForFinalSelection)throw new Error('Candidates for initial selection must be equal or more than candidates for final selection');\n\t\tif(candidatesForInitialSelection>candidates.length)throw new Error('There are '+candidatesForInitialSelection+'candidates for initial selection which is more than the total number of candidates of '+candidates.length);\n\t\tif(candidatesForFinalSelection>candidates.length)throw new Error('There are '+candidatesForFinalSelection+'candidates for initial selection which is more than the total number of candidates of '+candidates.length);\n\t\tif(!candidateIdentifierField){\n\t\t\tcandidateIdentifierField=Object.keys(candidates[0] as object)[0] as keyof Candidate;\n\t\t\tif(!candidateIdentifierField)throw new Error('No candidate identifier field');\n\t\t}\n\n\t\tlet rationale='';\n\t\tlet selectedCandidates:Candidate[]=[];\n\n\t\tawait this.checkEmbedderLoaded();\n\t\tif(!this.embedder)return;\n\t\tlet candidateDocuments=candidates.map((candidate,index)=>convertCandidateToDocument({\n\t\t\tcandidate,\n\t\t\tindex\n\t\t}));\n\t\tlet vectorStore=await MemoryVectorStore.fromTexts(\n\t\t\tlodash.cloneDeep(candidateDocuments),\n\t\t\tcandidateDocuments.map((document,index)=>index),\n\t\t\tthis\n\t\t);\n\n\t\tlet searchAreasPromptTemplate=this.generatePromptTemplate(generateSearchAreasInstruction(problemDescription));\n\t\tif(this.DEBUG)console.log('Formatted search areas prompt: '+searchAreasPromptTemplate);\n\t\tawait this.checkTokeniserLoaded();\n\t\tif(!this.tokeniser)return;\n\t\tlet searchAreasPromptTokens=this.tokeniser.encode(searchAreasPromptTemplate);\n\t\tif(searchAreasPromptTokens.length>this.tokeniser.model_max_length)throw new Error('Search areas instruction prompt is too long for the tokeniser model');\n\n\t\tawait this.checkGeneratorLoaded();\n\t\tif(!this.generator)return;\n\t\tlet pad_token_id=this.tokeniser.pad_token_id??this.tokeniser.sep_token_id??0;\n\t\tlet eos_token_id=this.tokeniser.sep_token_id??2;\n\t\tlet searchAreasReplyArray=await this.generator(searchAreasPromptTemplate,{\n\t\t\tmax_new_tokens:this.generateSearchAreasMaxNewTokens,\n\t\t\ttemperature:this.generateSearchAreasTemperature,\n\t\t\trepetition_penalty:this.generateSearchAreasRepetitionPenalty,\n\t\t\tpad_token_id,\n\t\t\teos_token_id\n\t\t});\n\t\tlet searchAreasReply=Array.isArray(searchAreasReplyArray?.[0])?searchAreasReplyArray?.[0]?.[0]:searchAreasReplyArray?.[0];\n\t\tif(!searchAreasReply.generated_text)throw new Error('No generated text for search areas');\n\t\tif(this.DEBUG)console.log('Generated search areas response: '+searchAreasReply.generated_text);\n\t\tlet searchAreasResponseIndex=searchAreasReply.generated_text.toString().indexOf('### Response:');\n\t\tif(searchAreasResponseIndex>=0)searchAreasResponseIndex+='### Response:'.length;\n\t\telse searchAreasResponseIndex=0;\n\n\t\tlet vectorSearchQuery=searchAreasReply.generated_text.toString().substring(searchAreasResponseIndex).trim();\n\t\t//generally the first sentence has the greatest relevance to the actual prompt\n\t\t//if(vectorSearchQuery.includes('.'))vectorSearchQuery=vectorSearchQuery.split('.')[0].trim();\n\t\tif(this.DEBUG)console.log('Vector search query: '+vectorSearchQuery);\n\t\tlet queryResult=await vectorStore.similaritySearch(vectorSearchQuery,candidatesForInitialSelection);\n\t\tif(this.DEBUG)console.log('Vector search results: ',queryResult);\n\n\t\tlet summaries:string[]=[];\n\t\t//only bother doing summarisation if there are candidates which exceed the token count\n\t\tif(queryResult.some(result=>result.pageContent.trim().split(/\\s+/).length>this.targetSummarisedStringTokenCount)){\n\t\t\tawait this.checkSummariserLoaded();\n\t\t\tif(!this.summariser)return;\n\t\t\tsummaries=(await Promise.allSettled(queryResult.map(async result=>{\n\t\t\t\tif(!result.pageContent||typeof result.pageContent!=='string')return '';\n\t\t\t\tif(result.pageContent.trim().split(/\\s+/).length<=this.targetSummarisedStringTokenCount)return result.pageContent;\n\t\t\t\tlet summarisableSubstringIndices:AICompareCandidates.SummarisableSubstringIndices={\n\t\t\t\t\tstart:0,\n\t\t\t\t\tend:result.pageContent.length\n\t\t\t\t};\n\t\t\t\tif(getSummarisableSubstringIndices)Object.assign(summarisableSubstringIndices,getSummarisableSubstringIndices(result.pageContent));\n\t\t\t\tsummarisableSubstringIndices.start=lodash.clamp(lodash.toSafeInteger(summarisableSubstringIndices.start),0,result.pageContent.length);\n\t\t\t\tsummarisableSubstringIndices.end=lodash.clamp(lodash.toSafeInteger(summarisableSubstringIndices.end),0,result.pageContent.length);\n\t\t\t\tlet summarisableSubstring=result.pageContent.substring(summarisableSubstringIndices.start,summarisableSubstringIndices.end);\n\t\t\t\tlet contentBefore=result.pageContent.substring(0,summarisableSubstringIndices.start);\n\t\t\t\tlet contentAfter=result.pageContent.substring(summarisableSubstringIndices.end);\n\t\t\t\tlet wordsWithoutSummarisable=contentBefore.split(/s+/).length+contentAfter.split(/s+/).length;\n\t\t\t\tlet targetSummarisedSubstringTokenCount=Math.max(1,420-wordsWithoutSummarisable);\n\t\t\t\tlet summarisedSubstringArray=await this.summariser?.(summarisableSubstring,<TextGenerationConfig>{\n\t\t\t\t\tmax_length:targetSummarisedSubstringTokenCount\n\t\t\t\t});\n\t\t\t\tlet summarisedSubstring=Array.isArray(summarisedSubstringArray?.[0])?summarisedSubstringArray?.[0]?.[0]:summarisedSubstringArray?.[0];\n\t\t\t\tlet summarisedString=contentBefore+(summarisedSubstring?.summary_text??'').split(/s+/).slice(targetSummarisedSubstringTokenCount).join(' ')+contentAfter;\n\t\t\t\tif(this.DEBUG)console.log('Summarised candidate: '+summarisedString);\n\t\t\t\treturn summarisedString;\n\t\t\t}))).filter(result=>result.status==='fulfilled'&&result.value).map(result=>(result as PromiseFulfilledResult<string>).value);\n\t\t}else{\n\t\t\tsummaries=queryResult.map(result=>result.pageContent);\n\t\t}\n\n\t\tlet rankingPromptTemplate=this.generatePromptTemplate(generateRankingInstruction({\n\t\t\tproblemDescription,\n\t\t\tsummaries,\n\t\t\tcandidatesForFinalSelection,\n\t\t\tcandidateIdentifierField:String(candidateIdentifierField)\n\t\t}));\n\t\tif(this.DEBUG)console.log('Formatted ranking prompt: '+rankingPromptTemplate);\n\t\tlet rankingPromptTokens=this.tokeniser.encode(rankingPromptTemplate);\n\t\tif(rankingPromptTokens.length>this.tokeniser.model_max_length)throw new Error('Ranking instruction prompt is too long for the tokeniser model');\n\t\tlet rankingArray=await this.generator(rankingPromptTemplate,{\n\t\t\tmax_new_tokens:this.rankingMaxNewTokens,\n\t\t\ttemperature:this.rankingTemperature,\n\t\t\trepetition_penalty:this.rankingRepetitionPenalty,\n\t\t\tpad_token_id,\n\t\t\teos_token_id\n\t\t});\n\t\tlet ranking=Array.isArray(rankingArray?.[0])?rankingArray?.[0]?.[0]:rankingArray[0];\n\t\trationale=ranking.generated_text.toString().trim().replace(/(\\*\\*)|(<\\/?s>)|(\\[.*?\\])\\s*/g, '');\n\t\tif(this.DEBUG)console.log('Generated rationale: '+rationale);\n\t\tlet rationaleResponseIndex=rationale.indexOf('### Response:');\n\t\tif(rationaleResponseIndex>=0)rationaleResponseIndex+='### Response:'.length;\n\t\telse rationaleResponseIndex=0;\n\t\trationale=rationale.substring(rationaleResponseIndex);\n\t\t//if(!rationale)throw new Error('No rationale generated');\n\n\t\tif(rationale){\n\t\t\tlet identifiers=extractIdentifiersFromRationale(rationale);\n\t\t\tif(identifiers.length>candidatesForFinalSelection)identifiers=identifiers.slice(0,candidatesForFinalSelection);\n\t\t\tselectedCandidates=lodash.compact(identifiers.map(identifier=>{\n\t\t\t\tlet selectedCandidate=candidates.find(candidate=>String(candidate[candidateIdentifierField]).toLowerCase()===identifier.toLowerCase());\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\tselectedCandidate=candidates.find(candidate=>String(candidate[candidateIdentifierField]).toLowerCase().includes(identifier.toLowerCase()));\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\tselectedCandidate=candidates.find(candidate=>identifier.toLowerCase().includes(String(candidate[candidateIdentifierField]).toLowerCase()));\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\treturn null;\n\t\t\t}));\n\t\t}\n\n\t\tif(!Array.isArray(selectedCandidates)||selectedCandidates.length<=0){\n\t\t\tselectedCandidates=lodash.uniq(lodash.compact(queryResult.map(result=>{\n\t\t\t\tlet identifier=extractIdentifierFromCandidateDocument({\n\t\t\t\t\tcandidateDocument:result.pageContent,\n\t\t\t\t\tcandidateIdentifierField:String(candidateIdentifierField)\n\t\t\t\t});\n\t\t\t\tif(this.DEBUG)console.log('Extracted identifier from candidate document: '+identifier);\n\t\t\t\tlet selectedCandidate=candidates.find(candidate=>String(candidate[candidateIdentifierField]).toLowerCase()===identifier.toLowerCase());\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\tselectedCandidate=candidates.find(candidate=>String(candidate[candidateIdentifierField]).toLowerCase().includes(identifier.toLowerCase()));\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\tselectedCandidate=candidates.find(candidate=>identifier.toLowerCase().includes(String(candidate[candidateIdentifierField]).toLowerCase()));\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\treturn null;\n\t\t\t}))).slice(candidatesForFinalSelection);\n\t\t}\n\t\tif(this.DEBUG)console.log('Selected candidates',selectedCandidates);\n\n\t\treturn{\n\t\t\trationale,\n\t\t\tselectedCandidates\n\t\t};\n\t}\n};\n\nexport namespace AICompareCandidates{\n\texport interface LoadArguments{\n\t\tprogressCallback?:ProgressCallback;\n\t\tmodelName?:string;\n\t};\n\n\texport interface SummarisableSubstringIndices{\n\t\tstart:number;\n\t\tend:number;\n\t};\n\n\texport interface CompareArguments<Candidate>{\n\t\tcandidates:Candidate[];\n\t\tproblemDescription:string;\n\t\tgenerateSearchAreasInstruction?:(problemDescription:string)=>string;\n\t\tconvertCandidateToDocument?:(convertCandidateToDocumentArguments:ConvertCandidateToDocumentArguments<Candidate>)=>string;\n\t\tcandidatesForInitialSelection?:number;\n\t\tcandidatesForFinalSelection?:number;\n\t\tgenerateRankingInstruction?:(generateRankingInstructionArguments:GenerateRankingInstructionArguments)=>string;\n\t\textractIdentifiersFromRationale?:(rationale:string)=>string[];\n\t\textractIdentifierFromCandidateDocument?:(extractIdentifierFromCandidateDocumentArguments:ExtractIdentifierFromCandidateDocumentArguments)=>string;\n\t\tcandidateIdentifierField?:keyof Candidate;\n\t\tgetSummarisableSubstringIndices?:(candidateDocument:string)=>SummarisableSubstringIndices;\n\t};\n\n\texport interface ConvertCandidateToDocumentArguments<Candidate>{\n\t\tcandidate:Candidate;\n\t\tindex:number;\n\t};\n\n\texport interface ExtractIdentifierFromCandidateDocumentArguments{\n\t\tcandidateDocument:string;\n\t\tcandidateIdentifierField:string;\n\t};\n\n\texport interface GenerateRankingInstructionArguments{\n\t\tproblemDescription:string;\n\t\tsummaries:string[];\n\t\tcandidatesForFinalSelection:number;\n\t\tcandidateIdentifierField:string;\n\t};\n\n\texport interface CompareCandidatesReturn<Candidate>{\n\t\tselectedCandidates:Candidate[],\n\t\trationale:string\n\t};\n};\n\nexport default AICompareCandidates;"],"mappings":"8PAiBA,IAAa,EAAb,cAAyC,CAAU,CAClD,IAAa,EACb,MAAM,GAEN,UAAsC,KACtC,mBAAmB,yBACnB,iBAAsD,KACtD,sBAAiD,EAAE,CACnD,0BAAgD,KAEhD,WAAsC,KACtC,oBAAoB,6BACpB,kBAAsD,KACtD,uBAAkD,EAAE,CACpD,2BAAiD,KAEjD,SAAwC,KACxC,kBAAkB,2BAClB,gBAAwD,KACxD,qBAAgD,EAAE,CAClD,yBAA+C,KAE/C,UAAmC,KACnC,mBAAmB,KAAK,mBACxB,iBAAmD,KACnD,sBAAiD,EAAE,CACnD,0BAAgD,KAEhD,gCAAgC,GAChC,+BAA+B,IAC/B,qCAAqC,IAErC,oBAAoB,GACpB,mBAAmB,IACnB,yBAAyB,IAEzB,iCAAiC,IAEjC,OACC,EAAI,eAAe,GACnB,EAAI,kBAAkB,GACtB,EAAI,iBAAiB,GAGtB,aAAa,CACZ,MAAM,EAAE,CAAC,CAGV,MAAM,cAAc,CACnB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAEzE,GADG,OAAO,GAAY,UAAU,IAAU,KAAK,mBAAmB,GAC/D,CAAC,KAAK,mBAAmB,MAAU,MAAM,+BAA+B,CAa3E,OAZG,IAAiB,KAAK,0BAA0B,GAGnD,KAAK,iBAAiB,EAAS,kBAAkB,KAAK,mBAAmB,CACxE,OAAO,SACP,kBAAkB,IACd,KAAK,OAAM,QAAQ,IAAI,EAAK,UAAU,EAAa,CAAC,CACvD,OAAO,OAAO,KAAK,sBAAsB,EAAa,CAC/C,KAAK,4BAA4B,EAAa,EAEtD,CAAC,CACF,KAAK,UAAU,MAAM,KAAK,iBACnB,KAAK,UAGb,MAAM,sBAAsB,CAG3B,GAFI,KAAK,kBAAiB,KAAK,eAAe,CAC1C,KAAK,WAAU,MAAM,KAAK,iBAC3B,CAAC,KAAK,UAAU,MAAU,MAAM,2BAA2B,CAG/D,MAAM,eAAe,CACpB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAEzE,GADG,OAAO,GAAY,UAAU,IAAU,KAAK,oBAAoB,GAChE,CAAC,KAAK,oBAAoB,MAAU,MAAM,gCAAgC,CAa7E,OAZG,IAAiB,KAAK,2BAA2B,GAGpD,KAAK,kBAAkB,EAAS,gBAAgB,KAAK,oBAAoB,CACxE,OAAO,SACP,kBAAkB,IACd,KAAK,OAAM,QAAQ,IAAI,EAAK,UAAU,EAAa,CAAC,CACvD,OAAO,OAAO,KAAK,uBAAuB,EAAa,CAChD,KAAK,6BAA6B,EAAa,EAEvD,CAAC,CACF,KAAK,WAAW,MAAM,KAAK,kBACpB,KAAK,WAGb,MAAM,uBAAuB,CAG5B,GAFI,KAAK,mBAAkB,KAAK,gBAAgB,CAC5C,KAAK,YAAW,MAAM,KAAK,kBAC5B,CAAC,KAAK,WAAW,MAAU,MAAM,4BAA4B,CAGjE,MAAM,aAAa,CAClB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAEzE,GADG,OAAO,GAAY,UAAU,IAAU,KAAK,kBAAkB,GAC9D,CAAC,KAAK,kBAAkB,MAAU,MAAM,8BAA8B,CAazE,OAZG,IAAiB,KAAK,yBAAyB,GAGlD,KAAK,gBAAgB,EAAS,qBAAqB,KAAK,kBAAkB,CACzE,OAAO,SACP,kBAAkB,IACd,KAAK,OAAM,QAAQ,IAAI,EAAK,UAAU,EAAa,CAAC,CACvD,OAAO,OAAO,KAAK,qBAAqB,EAAa,CAC9C,KAAK,2BAA2B,EAAa,EAErD,CAAC,CACF,KAAK,SAAS,MAAM,KAAK,gBAClB,KAAK,SAGb,MAAM,qBAAqB,CAG1B,GAFI,KAAK,iBAAgB,KAAK,cAAc,CACxC,KAAK,UAAS,MAAM,KAAK,gBAC1B,CAAC,KAAK,SAAS,MAAU,MAAM,0BAA0B,CAG7D,MAAM,cAAc,CACnB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAEzE,GADG,OAAO,GAAY,UAAU,IAAU,KAAK,mBAAmB,GAC/D,CAAC,KAAK,mBAAmB,MAAU,MAAM,+BAA+B,CAY3E,OAXG,IAAiB,KAAK,0BAA0B,GAGnD,KAAK,iBAAiB,EAAc,gBAAgB,KAAK,mBAAmB,CAC3E,kBAAkB,IACd,KAAK,OAAM,QAAQ,IAAI,EAAK,UAAU,EAAa,CAAC,CACvD,OAAO,OAAO,KAAK,sBAAsB,EAAa,CAC/C,KAAK,4BAA4B,EAAa,EAEtD,CAAC,CACF,KAAK,UAAU,MAAM,KAAK,iBACnB,KAAK,UAGb,MAAM,sBAAsB,CAG3B,GAFI,KAAK,kBAAiB,KAAK,eAAe,CAC1C,KAAK,WAAU,MAAM,KAAK,iBAC3B,CAAC,KAAK,UAAU,MAAU,MAAM,2BAA2B,CAG/D,MAAM,WAAW,EAA8B,CAE9C,OADA,MAAM,KAAK,qBAAqB,CACzB,MAAM,MAAM,MAAM,KAAK,WAAW,EAAK,CAC7C,QAAQ,OACR,UAAU,GACV,CAAC,GAAG,KAAK,CAGX,MAAM,eAAe,EAAmC,CACvD,OAAO,QAAQ,IAAI,EAAM,IAAI,GAAM,KAAK,WAAW,EAAK,CAAC,CAAC,CAG3D,uBAAuB,EAAc,CACpC,MAAO;;;EAEN,EACA;;eAGF,sCAAsC,EAA0B,CAC/D,MAAO;WAAyG,EAAmB,IAGpI,kCAA6C,CAC5C,YACA,SACwI,EAAE,CAAC,CAC3I,IAAI,EAAS,uBAAuB,EACpC,IAAI,IAAI,KAAK,EAAU,GAAU;EAAK,EAAO,UAAU,EAAE,CAAC,MAAM,OAAO,EAAU,IAAK,SAAS,EAAK,UAAU,EAAU,GAAG,CAAC,OAAO,EAAU,GAAG,EAEhJ,MADA,IAAU;oBAAuB,EAC1B,EAGR,kCAAkC,CACjC,qBACA,YACA,8BACA,4BACkH,EAAE,CAAC,CACrH,MAAO;uBACkB,EAA4B;mCAChB,EAAmB,QAAQ,WAAW,IAAI,CAAC;;;;;EAE7D,EAAU,KAAK;;EAAO,CAAC;;;YAE5B,EAAO,UAAU,EAAyB,CAAC;SAI1D,aAAa,EAAY,EAAa,EAAkB,CACpD,IAAI,EAAc,EAAK,MAAM,EAAW,CAAC,OAAO,EAAM,CACtD,OAAO,EAAc,EAAE,EAAc,EAAc,EAGvD,8CAA8C,CAC7C,oBACA,4BAC0I,EAAE,CAAC,CAC1I,KAAK,OAAM,QAAQ,IAAI,EAAkB,EAAyB,CACrE,IAAI,EAAU,EAAO,UAAU,EAAyB,CACpD,EAAW,EAAkB,QAAQ,EAAU,CAQnD,GAPG,EAAW,IAAE,EAAW,EAAkB,aAAa,CAAC,QAAQ,EAAU,aAAa,CAAC,EACxF,KAAK,OAAM,QAAQ,IAAI,EAAW,CAClC,GAAY,IAAE,GAAY,EAAU,QACpC,EAAW,IACb,EAAW,EAAkB,aAAa,CAAC,QAAQ,EAAyB,aAAa,CAAC,CACvF,GAAY,IAAE,GAAY,EAAyB,SAEpD,KAAK,MAAM,QAAQ,IAAI,EAAW,MAChC,MAAO,GAKZ,GAJA,EAAW,EAAkB,QAAQ,IAAI,EAAW,CACjD,KAAK,OAAM,QAAQ,IAAI,EAAW,CAClC,EAAW,IAAE,EAAW,KAAK,aAAa,EAAkB,MAAM,EAAW,EAC7E,KAAK,OAAM,QAAQ,IAAI,EAAW,CAClC,EAAW,EAAE,MAAO,GACvB,IAAI,EAAS,EAAkB,QAAQ;EAAK,EAAW,CAGvD,OAFG,EAAS,IAAE,EAAS,EAAkB,QACtC,KAAK,OAAM,QAAQ,IAAI,EAAS,CAC5B,EAAkB,UAAU,EAAW,EAAS,CAAC,MAAM,CAG/D,uCAAuC,EAAiB,CACvD,IAAI,EAAM,kCACNA,EAAiB,EAAE,CACvB,IAAI,IAAIC,EAA2B,MAAM,QAAQ,EAAM,EAAM,KAAK,EAAU,CAAC,EAAK,EAAM,IAAG,EAAQ,KAAK,EAAM,GAAG,CACjH,OAAO,EAGR,MAAM,kBAA6B,CAClC,aACA,qBAAmB,GACnB,iCAA+B,KAAK,sCAAsC,KAAK,KAAK,CACpF,6BAA2B,KAAK,kCAAkC,KAAK,KAAK,CAC5E,gCAA8B,EAC9B,8BAA4B,EAC5B,6BAA2B,KAAK,kCAAkC,KAAK,KAAK,CAC5E,kCAAgC,KAAK,uCAAuC,KAAK,KAAK,CACtF,yCAAuC,KAAK,8CAA8C,KAAK,KAAK,CACpG,2BAAyB,IAAA,GACzB,mCACkG,EAAE,CAAsE,CAC1K,GAAG,CAAC,MAAM,QAAQ,EAAW,EAAE,EAAW,QAAQ,EAAE,MAAU,MAAM,yBAAyB,CAI7F,GAHA,EAA8B,EAAO,cAAc,EAA8B,CAC9E,GAA+B,IAClC,EAA4B,EAAO,cAAc,EAA4B,CAC1E,GAA6B,GAAE,MAAU,MAAM,4EAA4E,CAC9H,GAAG,EAA8B,EAA4B,MAAU,MAAM,6FAA6F,CAC1K,GAAG,EAA8B,EAAW,OAAO,MAAU,MAAM,aAAa,EAA8B,yFAAyF,EAAW,OAAO,CACzN,GAAG,EAA4B,EAAW,OAAO,MAAU,MAAM,aAAa,EAA4B,yFAAyF,EAAW,OAAO,CACrN,GAAG,CAAC,IACH,EAAyB,OAAO,KAAK,EAAW,GAAa,CAAC,GAC3D,CAAC,GAAyB,MAAU,MAAM,gCAAgC,CAG9E,IAAI,EAAU,GACVC,EAA+B,EAAE,CAGrC,GADA,MAAM,KAAK,qBAAqB,CAC7B,CAAC,KAAK,SAAS,OAClB,IAAI,EAAmB,EAAW,KAAK,EAAU,IAAQ,EAA2B,CACnF,YACA,QACA,CAAC,CAAC,CACC,EAAY,MAAM,EAAkB,UACvC,EAAO,UAAU,EAAmB,CACpC,EAAmB,KAAK,EAAS,IAAQ,EAAM,CAC/C,KACA,CAEG,EAA0B,KAAK,uBAAuB,EAA+B,EAAmB,CAAC,CAG7G,GAFG,KAAK,OAAM,QAAQ,IAAI,kCAAkC,EAA0B,CACtF,MAAM,KAAK,sBAAsB,CAC9B,CAAC,KAAK,UAAU,OAEnB,GAD4B,KAAK,UAAU,OAAO,EAA0B,CACjD,OAAO,KAAK,UAAU,iBAAiB,MAAU,MAAM,sEAAsE,CAGxJ,GADA,MAAM,KAAK,sBAAsB,CAC9B,CAAC,KAAK,UAAU,OACnB,IAAI,EAAa,KAAK,UAAU,cAAc,KAAK,UAAU,cAAc,EACvE,EAAa,KAAK,UAAU,cAAc,EAC1C,EAAsB,MAAM,KAAK,UAAU,EAA0B,CACxE,eAAe,KAAK,gCACpB,YAAY,KAAK,+BACjB,mBAAmB,KAAK,qCACxB,eACA,eACA,CAAC,CACE,EAAiB,MAAM,QAAQ,IAAwB,GAAG,CAAC,IAAwB,KAAK,GAAG,IAAwB,GACvH,GAAG,CAAC,EAAiB,eAAe,MAAU,MAAM,qCAAqC,CACtF,KAAK,OAAM,QAAQ,IAAI,oCAAoC,EAAiB,eAAe,CAC9F,IAAI,EAAyB,EAAiB,eAAe,UAAU,CAAC,QAAQ,gBAAgB,CAC7F,GAA0B,EAAE,GAA0B,GACpD,EAAyB,EAE9B,IAAI,EAAkB,EAAiB,eAAe,UAAU,CAAC,UAAU,EAAyB,CAAC,MAAM,CAGxG,KAAK,OAAM,QAAQ,IAAI,wBAAwB,EAAkB,CACpE,IAAI,EAAY,MAAM,EAAY,iBAAiB,EAAkB,EAA8B,CAChG,KAAK,OAAM,QAAQ,IAAI,0BAA0B,EAAY,CAEhE,IAAIC,EAAmB,EAAE,CAEzB,GAAG,EAAY,KAAK,GAAQ,EAAO,YAAY,MAAM,CAAC,MAAM,MAAM,CAAC,OAAO,KAAK,iCAAiC,CAAC,CAEhH,GADA,MAAM,KAAK,uBAAuB,CAC/B,CAAC,KAAK,WAAW,OACpB,GAAW,MAAM,QAAQ,WAAW,EAAY,IAAI,KAAM,IAAQ,CACjE,GAAG,CAAC,EAAO,aAAa,OAAO,EAAO,aAAc,SAAS,MAAO,GACpE,GAAG,EAAO,YAAY,MAAM,CAAC,MAAM,MAAM,CAAC,QAAQ,KAAK,iCAAiC,OAAO,EAAO,YACtG,IAAIC,EAA8E,CACjF,MAAM,EACN,IAAI,EAAO,YAAY,OACvB,CACE,GAAgC,OAAO,OAAO,EAA6B,EAAgC,EAAO,YAAY,CAAC,CAClI,EAA6B,MAAM,EAAO,MAAM,EAAO,cAAc,EAA6B,MAAM,CAAC,EAAE,EAAO,YAAY,OAAO,CACrI,EAA6B,IAAI,EAAO,MAAM,EAAO,cAAc,EAA6B,IAAI,CAAC,EAAE,EAAO,YAAY,OAAO,CACjI,IAAI,EAAsB,EAAO,YAAY,UAAU,EAA6B,MAAM,EAA6B,IAAI,CACvH,EAAc,EAAO,YAAY,UAAU,EAAE,EAA6B,MAAM,CAChF,EAAa,EAAO,YAAY,UAAU,EAA6B,IAAI,CAC3E,EAAyB,EAAc,MAAM,KAAK,CAAC,OAAO,EAAa,MAAM,KAAK,CAAC,OACnF,EAAoC,KAAK,IAAI,EAAE,IAAI,EAAyB,CAC5E,EAAyB,MAAM,KAAK,aAAa,EAA4C,CAChG,WAAW,EACX,CAAC,CAEE,EAAiB,IADG,MAAM,QAAQ,IAA2B,GAAG,CAAC,IAA2B,KAAK,GAAG,IAA2B,KAC1E,cAAc,IAAI,MAAM,KAAK,CAAC,MAAM,EAAoC,CAAC,KAAK,IAAI,CAAC,EAE5I,OADG,KAAK,OAAM,QAAQ,IAAI,yBAAyB,EAAiB,CAC7D,GACN,CAAC,EAAE,OAAO,GAAQ,EAAO,SAAS,aAAa,EAAO,MAAM,CAAC,IAAI,GAAS,EAA0C,MAAM,MAE5H,EAAU,EAAY,IAAI,GAAQ,EAAO,YAAY,CAGtD,IAAI,EAAsB,KAAK,uBAAuB,EAA2B,CAChF,qBACA,YACA,8BACA,yBAAyB,OAAO,EAAyB,CACzD,CAAC,CAAC,CAGH,GAFG,KAAK,OAAM,QAAQ,IAAI,6BAA6B,EAAsB,CACrD,KAAK,UAAU,OAAO,EAAsB,CAC7C,OAAO,KAAK,UAAU,iBAAiB,MAAU,MAAM,iEAAiE,CAC/I,IAAI,EAAa,MAAM,KAAK,UAAU,EAAsB,CAC3D,eAAe,KAAK,oBACpB,YAAY,KAAK,mBACjB,mBAAmB,KAAK,yBACxB,eACA,eACA,CAAC,CAEF,GADY,MAAM,QAAQ,IAAe,GAAG,CAAC,IAAe,KAAK,GAAG,EAAa,IAC/D,eAAe,UAAU,CAAC,MAAM,CAAC,QAAQ,gCAAiC,GAAG,CAC5F,KAAK,OAAM,QAAQ,IAAI,wBAAwB,EAAU,CAC5D,IAAI,EAAuB,EAAU,QAAQ,gBAAgB,CAM7D,GALG,GAAwB,EAAE,GAAwB,GAChD,EAAuB,EAC5B,EAAU,EAAU,UAAU,EAAuB,CAGlD,EAAU,CACZ,IAAI,EAAY,EAAgC,EAAU,CACvD,EAAY,OAAO,IAA4B,EAAY,EAAY,MAAM,EAAE,EAA4B,EAC9G,EAAmB,EAAO,QAAQ,EAAY,IAAI,GAAY,CAC7D,IAAI,EAAkB,EAAW,KAAK,GAAW,OAAO,EAAU,GAA0B,CAAC,aAAa,GAAG,EAAW,aAAa,CAAC,CAMtI,OALG,IACH,EAAkB,EAAW,KAAK,GAAW,OAAO,EAAU,GAA0B,CAAC,aAAa,CAAC,SAAS,EAAW,aAAa,CAAC,CAAC,CACvI,KACH,EAAkB,EAAW,KAAK,GAAW,EAAW,aAAa,CAAC,SAAS,OAAO,EAAU,GAA0B,CAAC,aAAa,CAAC,CAAC,CACvI,GAAyB,EACrB,MACN,CAAC,CAqBJ,OAlBG,CAAC,MAAM,QAAQ,EAAmB,EAAE,EAAmB,QAAQ,KACjE,EAAmB,EAAO,KAAK,EAAO,QAAQ,EAAY,IAAI,GAAQ,CACrE,IAAI,EAAW,EAAuC,CACrD,kBAAkB,EAAO,YACzB,yBAAyB,OAAO,EAAyB,CACzD,CAAC,CACC,KAAK,OAAM,QAAQ,IAAI,iDAAiD,EAAW,CACtF,IAAI,EAAkB,EAAW,KAAK,GAAW,OAAO,EAAU,GAA0B,CAAC,aAAa,GAAG,EAAW,aAAa,CAAC,CAMtI,OALG,IACH,EAAkB,EAAW,KAAK,GAAW,OAAO,EAAU,GAA0B,CAAC,aAAa,CAAC,SAAS,EAAW,aAAa,CAAC,CAAC,CACvI,KACH,EAAkB,EAAW,KAAK,GAAW,EAAW,aAAa,CAAC,SAAS,OAAO,EAAU,GAA0B,CAAC,aAAa,CAAC,CAAC,CACvI,GAAyB,EACrB,MACN,CAAC,CAAC,CAAC,MAAM,EAA4B,EAErC,KAAK,OAAM,QAAQ,IAAI,sBAAsB,EAAmB,CAE7D,CACL,YACA,qBACA,2BAoDH,IAAA,EAAe"}
+{"version":3,"file":"index.mjs","names":["matches:string[]","match:RegExpExecArray|null","selectedCandidates:Candidate[]","summaries:string[]","summarisableSubstringIndices:AICompareCandidates.SummarisableSubstringIndices"],"sources":["../src/index.ts"],"sourcesContent":["import{\n\tenv,\n\tpipeline,\n\tAutoTokenizer,\n\tTextGenerationPipeline,\n\tProgressInfo,\n\tProgressCallback,\n\tSummarizationPipeline,\n\tFeatureExtractionPipeline,\n\tPreTrainedTokenizer,\n\tTextGenerationConfig\n}from '@sroussey/transformers';\nimport {MemoryVectorStore} from '@langchain/classic/vectorstores/memory';\nimport {Embeddings} from '@langchain/core/embeddings';\nimport lodash from 'lodash';\nimport jsan from 'jsan';\n\nexport class AICompareCandidates extends Embeddings{\n\treadonly env=env;\n\tDEBUG=true;\n\n\tgenerator:TextGenerationPipeline|null=null;\n\tgeneratorModelName='Xenova/LaMini-GPT-774M';\n\tgeneratorPromise:Promise<TextGenerationPipeline>|null=null;\n\tgeneratorProgressInfo:ProgressInfo=<ProgressInfo>{};\n\tgeneratorProgressCallback:ProgressCallback|null=null;\n\tgeneratorAbortController=new AbortController();\n\n\tsummariser:SummarizationPipeline|null=null;\n\tsummariserModelName='Xenova/distilbart-cnn-12-6';\n\tsummariserPromise:Promise<SummarizationPipeline>|null=null;\n\tsummariserProgressInfo:ProgressInfo=<ProgressInfo>{};\n\tsummariserProgressCallback:ProgressCallback|null=null;\n\tsummariserAbortController=new AbortController();\n\n\tembedder:FeatureExtractionPipeline|null=null;\n\tembedderModelName='Xenova/all-MiniLM-L12-v2';\n\tembedderPromise:Promise<FeatureExtractionPipeline>|null=null;\n\tembedderProgressInfo:ProgressInfo=<ProgressInfo>{};\n\tembedderProgressCallback:ProgressCallback|null=null;\n\tembedderAbortController=new AbortController();\n\n\ttokeniser:PreTrainedTokenizer|null=null;\n\ttokeniserModelName=this.generatorModelName;\n\ttokeniserPromise:Promise<PreTrainedTokenizer>|null=null;\n\ttokeniserProgressInfo:ProgressInfo=<ProgressInfo>{};\n\ttokeniserProgressCallback:ProgressCallback|null=null;\n\ttokeniserAbortController=new AbortController();\n\n\tgenerateSearchAreasMaxNewTokens=64;\n\tgenerateSearchAreasTemperature=0.35;\n\tgenerateSearchAreasRepetitionPenalty=1.5;\n\n\trankingMaxNewTokens=64;\n\trankingTemperature=0.35;\n\trankingRepetitionPenalty=1.5;\n\n\ttargetSummarisedStringTokenCount=420;\n\n\tstatic{\n\t\tenv.localModelPath='';\n\t\tenv.allowRemoteModels=true;\n\t\tenv.allowLocalModels=false;\n\t}\n\n\tconstructor(){\n\t\tsuper({});\n\t}\n\t\n\tasync loadGenerator({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(typeof modelName==='string'&&modelName)this.generatorModelName=modelName;\n\t\tif(!this.generatorModelName)throw new Error('Invalid generator model name');\n\t\tif(progressCallback)this.generatorProgressCallback=progressCallback;\n\t\t//ts-ignore is needed due to frequent error TS2590: Expression produces a union type that is too complex to represent.\n\t\t//@ts-ignore\n\t\tthis.generatorPromise=pipeline('text-generation',this.generatorModelName,{\n\t\t\tdevice:'webgpu',\n\t\t\tprogress_callback:progressInfo=>{\n\t\t\t\tif(this.DEBUG)console.log(jsan.stringify(progressInfo));\n\t\t\t\tObject.assign(this.generatorProgressInfo,progressInfo);\n\t\t\t\treturn this.generatorProgressCallback?.(progressInfo);\n\t\t\t},\n\t\t\tabort_signal:this.generatorAbortController.signal\n\t\t});\n\t\tthis.generator=await this.generatorPromise;\n\t\treturn this.generator;\n\t}\n\n\tasync checkGeneratorLoaded({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(!this.generatorPromise)this.loadGenerator({\n\t\t\tprogressCallback,\n\t\t\tmodelName\n\t\t});\n\t\tif(!this.generator){\n\t\t\ttry{\n\t\t\t\tawait this.generatorPromise;\n\t\t\t}catch(error){\n\t\t\t\tthis.generatorPromise=null;\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t}\n\t\tif(!this.generator){\n\t\t\tthis.generatorPromise=null;\n\t\t\tthrow new Error('Unable to load generator');\n\t\t}\n\t}\n\n\tasync abortLoadGenerator(reason?:any){\n\t\tthis.generatorAbortController.abort(reason);\n\t\tthis.generatorAbortController=new AbortController();\n\t}\n\n\tasync loadSummariser({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(typeof modelName==='string'&&modelName)this.summariserModelName=modelName;\n\t\tif(!this.summariserModelName)throw new Error('Invalid summariser model name');\n\t\tif(progressCallback)this.summariserProgressCallback=progressCallback;\n\t\t//ts-ignore is needed due to frequent error TS2590: Expression produces a union type that is too complex to represent.\n\t\t//@ts-ignore\n\t\tthis.summariserPromise=pipeline('summarization',this.summariserModelName,{\n\t\t\tdevice:'webgpu',\n\t\t\tprogress_callback:progressInfo=>{\n\t\t\t\tif(this.DEBUG)console.log(jsan.stringify(progressInfo));\n\t\t\t\tObject.assign(this.summariserProgressInfo,progressInfo);\n\t\t\t\treturn this.summariserProgressCallback?.(progressInfo);\n\t\t\t},\n\t\t\tabort_signal:this.summariserAbortController.signal\n\t\t});\n\t\tthis.summariser=await this.summariserPromise;\n\t\treturn this.summariser;\n\t}\n\n\tasync checkSummariserLoaded({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(!this.summariserPromise)this.loadSummariser({\n\t\t\tprogressCallback,\n\t\t\tmodelName\n\t\t});\n\t\tif(!this.summariser){\n\t\t\ttry{\n\t\t\t\tawait this.summariserPromise;\n\t\t\t}catch(error){\n\t\t\t\tthis.summariserPromise=null;\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t}\n\t\tif(!this.summariser){\n\t\t\tthis.summariserPromise=null;\n\t\t\tthrow new Error('Unable to load summariser');\n\t\t}\n\t}\n\n\tasync abortLoadSummariser(){\n\t\tthis.summariserAbortController.abort();\n\t\tthis.summariserAbortController=new AbortController();\n\t}\n\n\tasync loadEmbedder({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(typeof modelName==='string'&&modelName)this.embedderModelName=modelName;\n\t\tif(!this.embedderModelName)throw new Error('Invalid embedder model name');\n\t\tif(progressCallback)this.embedderProgressCallback=progressCallback;\n\t\t//ts-ignore is needed due to frequent error TS2590: Expression produces a union type that is too complex to represent.\n\t\t//@ts-ignore\n\t\tthis.embedderPromise=pipeline('feature-extraction',this.embedderModelName,{\n\t\t\tdevice:'webgpu',\n\t\t\tprogress_callback:progressInfo=>{\n\t\t\t\tif(this.DEBUG)console.log(jsan.stringify(progressInfo));\n\t\t\t\tObject.assign(this.embedderProgressInfo,progressInfo);\n\t\t\t\treturn this.embedderProgressCallback?.(progressInfo);\n\t\t\t},\n\t\t\tabort_signal:this.embedderAbortController.signal\n\t\t});\n\t\tthis.embedder=await this.embedderPromise;\n\t\treturn this.embedder;\n\t}\n\n\tasync checkEmbedderLoaded({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(!this.embedderPromise)this.loadEmbedder({\n\t\t\tprogressCallback,\n\t\t\tmodelName\n\t\t});\n\t\tif(!this.embedder){\n\t\t\ttry{\n\t\t\t\tawait this.embedderPromise;\n\t\t\t}catch(error){\n\t\t\t\tthis.embedderPromise=null;\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t}\n\t\tif(!this.embedder){\n\t\t\tthis.embedderPromise=null;\n\t\t\tthrow new Error('Unable to load embedder');\n\t\t}\n\t}\n\n\tasync abortLoadEmbedder(){\n\t\tthis.embedderAbortController.abort();\n\t\tthis.embedderAbortController=new AbortController();\n\t}\n\n\tasync loadTokeniser({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(typeof modelName==='string'&&modelName)this.tokeniserModelName=modelName;\n\t\tif(!this.tokeniserModelName)throw new Error('Invalid tokeniser model name');\n\t\tif(progressCallback)this.tokeniserProgressCallback=progressCallback;\n\t\t//ts-ignore is needed due to frequent error TS2590: Expression produces a union type that is too complex to represent.\n\t\t//@ts-ignore\n\t\tthis.tokeniserPromise=AutoTokenizer.from_pretrained(this.tokeniserModelName,{\n\t\t\tprogress_callback:progressInfo=>{\n\t\t\t\tif(this.DEBUG)console.log(jsan.stringify(progressInfo));\n\t\t\t\tObject.assign(this.tokeniserProgressInfo,progressInfo);\n\t\t\t\treturn this.tokeniserProgressCallback?.(progressInfo);\n\t\t\t},\n\t\t\tabort_signal:this.tokeniserAbortController.signal\n\t\t});\n\t\tthis.tokeniser=await this.tokeniserPromise;\n\t\treturn this.tokeniser;\n\t}\n\n\tasync checkTokeniserLoaded({\n\t\tprogressCallback,\n\t\tmodelName=''\n\t}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){\n\t\tif(!this.tokeniserPromise)this.loadTokeniser({\n\t\t\tprogressCallback,\n\t\t\tmodelName\n\t\t});\n\t\tif(!this.tokeniser){\n\t\t\ttry{\n\t\t\t\tawait this.tokeniserPromise;\n\t\t\t}catch(error){\n\t\t\t\tthis.tokeniserPromise=null;\n\t\t\t\tthrow error;\n\t\t\t}\n\t\t}\n\t\tif(!this.tokeniser){\n\t\t\tthis.tokeniserPromise=null;\n\t\t\tthrow new Error('Unable to load tokeniser');\n\t\t}\n\t}\n\n\tasync abortLoadTokeniser(){\n\t\tthis.tokeniserAbortController.abort();\n\t\tthis.tokeniserAbortController=new AbortController();\n\t}\n\n\tasync embedQuery(text:string):Promise<number[]>{\n\t\tawait this.checkEmbedderLoaded();\n\t\treturn Array.from((await this.embedder?.(text,{\n\t\t\tpooling:'mean',\n\t\t\tnormalize:true\n\t\t}))?.data);\n\t}\n\n\tasync embedDocuments(texts:string[]):Promise<number[][]>{\n\t\treturn Promise.all(texts.map(text=>this.embedQuery(text)));\n\t}\n\n\tgeneratePromptTemplate(prompt:string){\n\t\treturn 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n'+\n\t\t\t'### Instruction:\\n'+\n\t\t\tprompt+\n\t\t\t'\\n\\n### Response:';\n\t}\n\n\tdefaultGenerateSearchAreasInstruction(problemDescription:string){\n\t\treturn 'List the relevant subject areas for the following issues. Limit your response to 100 words.\\nIssues: \"'+problemDescription+'\"';\n\t}\n\n\tdefaultConvertCandidateToDocument<Candidate>({\n\t\tcandidate,\n\t\tindex\n\t}:AICompareCandidates.ConvertCandidateToDocumentArguments<Candidate>=<AICompareCandidates.ConvertCandidateToDocumentArguments<Candidate>>{}){\n\t\tlet document='Start of Candidate #'+index;\n\t\tfor(let i in candidate)document+='\\n'+lodash.startCase(i)+': '+(typeof candidate[i]==='object'?jsan.stringify(candidate[i]):String(candidate[i]));\n\t\tdocument+='\\nEnd of Candidate #'+index;\n\t\treturn document;\n\t}\n\n\tdefaultGenerateRankingInstruction({\n\t\tproblemDescription,\n\t\tsummaries,\n\t\tcandidatesForFinalSelection,\n\t\tcandidateIdentifierField\n\t}:AICompareCandidates.GenerateRankingInstructionArguments=<AICompareCandidates.GenerateRankingInstructionArguments>{}){\n\t\treturn 'Strictly follow these rules:\\n'+\n\t\t\t'1. Rank ONLY the top '+candidatesForFinalSelection+' candidates with one 15-word sentence explaining why\\n'+\n\t\t\t'2. Rank the candidates based on \"'+problemDescription.replace(/(\\r|\\n)/g,' ')+'\"\\n'+\n\t\t\t'3. If unclear, say \"Insufficient information to determine\"\\n\\n'+\n\t\t\t'Candidates:\\n\\n'+summaries.join('\\n\\n')+'\\n\\n'+\n\t\t\t'Format exactly:\\n'+\n\t\t\t'#1. \"Full '+lodash.startCase(candidateIdentifierField)+'\": 15-word explanation\\n'+\n\t\t\t'#2. ...';\n\t}\n\n\tregexIndexOf(text:string,regex:RegExp,startIndex:number){\n    \tlet indexInSuffix=text.slice(startIndex).search(regex);\n    \treturn indexInSuffix<0?indexInSuffix:indexInSuffix+startIndex;\n\t}\n\n\tdefaultExtractIdentifierFromCandidateDocument({\n\t\tcandidateDocument,\n\t\tcandidateIdentifierField\n\t}:AICompareCandidates.ExtractIdentifierFromCandidateDocumentArguments=<AICompareCandidates.ExtractIdentifierFromCandidateDocumentArguments>{}){\n\t\tif(this.DEBUG)console.log(candidateDocument,candidateIdentifierField);\n\t\tlet startCase=lodash.startCase(candidateIdentifierField);\n\t\tlet startIndex=candidateDocument.indexOf(startCase);\n\t\tif(startIndex<0)startIndex=candidateDocument.toLowerCase().indexOf(startCase.toLowerCase());\n\t\tif(this.DEBUG)console.log(startIndex);\n\t\tif(startIndex>=0)startIndex+=startCase.length;\n\t\tif(startIndex<0){\n\t\t\tstartIndex=candidateDocument.toLowerCase().indexOf(candidateIdentifierField.toLowerCase());\n\t\t\tif(startIndex>=0)startIndex+=candidateIdentifierField.length;\n\t\t}\n\t\tif(this.DEBUG)console.log(startIndex);\n\t\telse return '';\n\t\tstartIndex=candidateDocument.indexOf(':',startIndex);\n\t\tif(this.DEBUG)console.log(startIndex);\n\t\tif(startIndex<0)startIndex=this.regexIndexOf(candidateDocument,/\\s+/,startIndex);\n\t\tif(this.DEBUG)console.log(startIndex);\n\t\tif(startIndex<0)return '';\n\t\tlet endIndex=candidateDocument.indexOf('\\n',startIndex);\n\t\tif(endIndex<0)endIndex=candidateDocument.length;\n\t\tif(this.DEBUG)console.log(endIndex);\n\t\treturn candidateDocument.substring(startIndex,endIndex).trim();\n\t}\n\n\tdefaultExtractIdentifiersFromRationale(rationale:string){\n\t\tlet regex=/^\\s*#\\s*\\d+\\s*\\.?\\s*\"([^\"]+)\"/gm;\n\t\tlet matches:string[]=[];\n\t\tfor(let match:RegExpExecArray|null;Array.isArray(match=regex.exec(rationale));)if(match[1])matches.push(match[1]);\n\t\treturn matches;\n\t}\n\n\tasync compareCandidates<Candidate>({\n\t\tcandidates,\n\t\tproblemDescription='',\n\t\tgenerateSearchAreasInstruction=this.defaultGenerateSearchAreasInstruction.bind(this),\n\t\tconvertCandidateToDocument=this.defaultConvertCandidateToDocument.bind(this),\n\t\tcandidatesForInitialSelection=2,\n\t\tcandidatesForFinalSelection=1,\n\t\tgenerateRankingInstruction=this.defaultGenerateRankingInstruction.bind(this),\n\t\textractIdentifiersFromRationale=this.defaultExtractIdentifiersFromRationale.bind(this),\n\t\textractIdentifierFromCandidateDocument=this.defaultExtractIdentifierFromCandidateDocument.bind(this),\n\t\tcandidateIdentifierField=undefined,\n\t\tgetSummarisableSubstringIndices\n\t}:AICompareCandidates.CompareArguments<Candidate>=<AICompareCandidates.CompareArguments<Candidate>>{}):Promise<AICompareCandidates.CompareCandidatesReturn<Candidate>|void>{\n\t\tif(!Array.isArray(candidates)||candidates.length<=0)throw new Error('No candidates provided');\n\t\tcandidatesForInitialSelection=lodash.toSafeInteger(candidatesForInitialSelection);\n\t\tif(candidatesForInitialSelection<=0)throw new Error('Candidates for initial selection must be a positive integer bigger than 0');\n\t\tcandidatesForFinalSelection=lodash.toSafeInteger(candidatesForFinalSelection);\n\t\tif(candidatesForFinalSelection<=0)throw new Error('Candidates for initial selection must be a positive integer bigger than 0');\n\t\tif(candidatesForInitialSelection<candidatesForFinalSelection)throw new Error('Candidates for initial selection must be equal or more than candidates for final selection');\n\t\tif(candidatesForInitialSelection>candidates.length)throw new Error('There are '+candidatesForInitialSelection+'candidates for initial selection which is more than the total number of candidates of '+candidates.length);\n\t\tif(candidatesForFinalSelection>candidates.length)throw new Error('There are '+candidatesForFinalSelection+'candidates for initial selection which is more than the total number of candidates of '+candidates.length);\n\t\tif(!candidateIdentifierField){\n\t\t\tcandidateIdentifierField=Object.keys(candidates[0] as object)[0] as keyof Candidate;\n\t\t\tif(!candidateIdentifierField)throw new Error('No candidate identifier field');\n\t\t}\n\n\t\tlet rationale='';\n\t\tlet selectedCandidates:Candidate[]=[];\n\n\t\tawait this.checkEmbedderLoaded();\n\t\tif(!this.embedder)return;\n\t\tlet candidateDocuments=candidates.map((candidate,index)=>convertCandidateToDocument({\n\t\t\tcandidate,\n\t\t\tindex\n\t\t}));\n\t\tlet vectorStore=await MemoryVectorStore.fromTexts(\n\t\t\tlodash.cloneDeep(candidateDocuments),\n\t\t\tcandidateDocuments.map((document,index)=>index),\n\t\t\tthis\n\t\t);\n\n\t\tlet searchAreasPromptTemplate=this.generatePromptTemplate(generateSearchAreasInstruction(problemDescription));\n\t\tif(this.DEBUG)console.log('Formatted search areas prompt: '+searchAreasPromptTemplate);\n\t\tawait this.checkTokeniserLoaded();\n\t\tif(!this.tokeniser)return;\n\t\tlet searchAreasPromptTokens=this.tokeniser.encode(searchAreasPromptTemplate);\n\t\tif(searchAreasPromptTokens.length>this.tokeniser.model_max_length)throw new Error('Search areas instruction prompt is too long for the tokeniser model');\n\n\t\tawait this.checkGeneratorLoaded();\n\t\tif(!this.generator)return;\n\t\tlet pad_token_id=this.tokeniser.pad_token_id??this.tokeniser.sep_token_id??0;\n\t\tlet eos_token_id=this.tokeniser.sep_token_id??2;\n\t\tlet searchAreasReplyArray=await this.generator(searchAreasPromptTemplate,{\n\t\t\tmax_new_tokens:this.generateSearchAreasMaxNewTokens,\n\t\t\ttemperature:this.generateSearchAreasTemperature,\n\t\t\trepetition_penalty:this.generateSearchAreasRepetitionPenalty,\n\t\t\tpad_token_id,\n\t\t\teos_token_id\n\t\t});\n\t\tlet searchAreasReply=Array.isArray(searchAreasReplyArray?.[0])?searchAreasReplyArray?.[0]?.[0]:searchAreasReplyArray?.[0];\n\t\tif(!searchAreasReply.generated_text)throw new Error('No generated text for search areas');\n\t\tif(this.DEBUG)console.log('Generated search areas response: '+searchAreasReply.generated_text);\n\t\tlet searchAreasResponseIndex=searchAreasReply.generated_text.toString().indexOf('### Response:');\n\t\tif(searchAreasResponseIndex>=0)searchAreasResponseIndex+='### Response:'.length;\n\t\telse searchAreasResponseIndex=0;\n\n\t\tlet vectorSearchQuery=searchAreasReply.generated_text.toString().substring(searchAreasResponseIndex).trim();\n\t\t//generally the first sentence has the greatest relevance to the actual prompt\n\t\t//if(vectorSearchQuery.includes('.'))vectorSearchQuery=vectorSearchQuery.split('.')[0].trim();\n\t\tif(this.DEBUG)console.log('Vector search query: '+vectorSearchQuery);\n\t\tlet queryResult=await vectorStore.similaritySearch(vectorSearchQuery,candidatesForInitialSelection);\n\t\tif(this.DEBUG)console.log('Vector search results: ',queryResult);\n\n\t\tlet summaries:string[]=[];\n\t\t//only bother doing summarisation if there are candidates which exceed the token count\n\t\tif(queryResult.some(result=>result.pageContent.trim().split(/\\s+/).length>this.targetSummarisedStringTokenCount)){\n\t\t\tawait this.checkSummariserLoaded();\n\t\t\tif(!this.summariser)return;\n\t\t\tsummaries=(await Promise.allSettled(queryResult.map(async result=>{\n\t\t\t\tif(!result.pageContent||typeof result.pageContent!=='string')return '';\n\t\t\t\tif(result.pageContent.trim().split(/\\s+/).length<=this.targetSummarisedStringTokenCount)return result.pageContent;\n\t\t\t\tlet summarisableSubstringIndices:AICompareCandidates.SummarisableSubstringIndices={\n\t\t\t\t\tstart:0,\n\t\t\t\t\tend:result.pageContent.length\n\t\t\t\t};\n\t\t\t\tif(getSummarisableSubstringIndices)Object.assign(summarisableSubstringIndices,getSummarisableSubstringIndices(result.pageContent));\n\t\t\t\tsummarisableSubstringIndices.start=lodash.clamp(lodash.toSafeInteger(summarisableSubstringIndices.start),0,result.pageContent.length);\n\t\t\t\tsummarisableSubstringIndices.end=lodash.clamp(lodash.toSafeInteger(summarisableSubstringIndices.end),0,result.pageContent.length);\n\t\t\t\tlet summarisableSubstring=result.pageContent.substring(summarisableSubstringIndices.start,summarisableSubstringIndices.end);\n\t\t\t\tlet contentBefore=result.pageContent.substring(0,summarisableSubstringIndices.start);\n\t\t\t\tlet contentAfter=result.pageContent.substring(summarisableSubstringIndices.end);\n\t\t\t\tlet wordsWithoutSummarisable=contentBefore.split(/s+/).length+contentAfter.split(/s+/).length;\n\t\t\t\tlet targetSummarisedSubstringTokenCount=Math.max(1,420-wordsWithoutSummarisable);\n\t\t\t\tlet summarisedSubstringArray=await this.summariser?.(summarisableSubstring,<TextGenerationConfig>{\n\t\t\t\t\tmax_length:targetSummarisedSubstringTokenCount\n\t\t\t\t});\n\t\t\t\tlet summarisedSubstring=Array.isArray(summarisedSubstringArray?.[0])?summarisedSubstringArray?.[0]?.[0]:summarisedSubstringArray?.[0];\n\t\t\t\tlet summarisedString=contentBefore+(summarisedSubstring?.summary_text??'').split(/s+/).slice(targetSummarisedSubstringTokenCount).join(' ')+contentAfter;\n\t\t\t\tif(this.DEBUG)console.log('Summarised candidate: '+summarisedString);\n\t\t\t\treturn summarisedString;\n\t\t\t}))).filter(result=>result.status==='fulfilled'&&result.value).map(result=>(result as PromiseFulfilledResult<string>).value);\n\t\t}else{\n\t\t\tsummaries=queryResult.map(result=>result.pageContent);\n\t\t}\n\n\t\tlet rankingPromptTemplate=this.generatePromptTemplate(generateRankingInstruction({\n\t\t\tproblemDescription,\n\t\t\tsummaries,\n\t\t\tcandidatesForFinalSelection,\n\t\t\tcandidateIdentifierField:String(candidateIdentifierField)\n\t\t}));\n\t\tif(this.DEBUG)console.log('Formatted ranking prompt: '+rankingPromptTemplate);\n\t\tlet rankingPromptTokens=this.tokeniser.encode(rankingPromptTemplate);\n\t\tif(rankingPromptTokens.length>this.tokeniser.model_max_length)throw new Error('Ranking instruction prompt is too long for the tokeniser model');\n\t\tlet rankingArray=await this.generator(rankingPromptTemplate,{\n\t\t\tmax_new_tokens:this.rankingMaxNewTokens,\n\t\t\ttemperature:this.rankingTemperature,\n\t\t\trepetition_penalty:this.rankingRepetitionPenalty,\n\t\t\tpad_token_id,\n\t\t\teos_token_id\n\t\t});\n\t\tlet ranking=Array.isArray(rankingArray?.[0])?rankingArray?.[0]?.[0]:rankingArray[0];\n\t\trationale=ranking.generated_text.toString().trim().replace(/(\\*\\*)|(<\\/?s>)|(\\[.*?\\])\\s*/g, '');\n\t\tif(this.DEBUG)console.log('Generated rationale: '+rationale);\n\t\tlet rationaleResponseIndex=rationale.indexOf('### Response:');\n\t\tif(rationaleResponseIndex>=0)rationaleResponseIndex+='### Response:'.length;\n\t\telse rationaleResponseIndex=0;\n\t\trationale=rationale.substring(rationaleResponseIndex);\n\t\t//if(!rationale)throw new Error('No rationale generated');\n\n\t\tif(rationale){\n\t\t\tlet identifiers=extractIdentifiersFromRationale(rationale);\n\t\t\tif(identifiers.length>candidatesForFinalSelection)identifiers=identifiers.slice(0,candidatesForFinalSelection);\n\t\t\tselectedCandidates=lodash.compact(identifiers.map(identifier=>{\n\t\t\t\tlet selectedCandidate=candidates.find(candidate=>String(candidate[candidateIdentifierField]).toLowerCase()===identifier.toLowerCase());\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\tselectedCandidate=candidates.find(candidate=>String(candidate[candidateIdentifierField]).toLowerCase().includes(identifier.toLowerCase()));\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\tselectedCandidate=candidates.find(candidate=>identifier.toLowerCase().includes(String(candidate[candidateIdentifierField]).toLowerCase()));\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\treturn null;\n\t\t\t}));\n\t\t}\n\n\t\tif(!Array.isArray(selectedCandidates)||selectedCandidates.length<=0){\n\t\t\tselectedCandidates=lodash.uniq(lodash.compact(queryResult.map(result=>{\n\t\t\t\tlet identifier=extractIdentifierFromCandidateDocument({\n\t\t\t\t\tcandidateDocument:result.pageContent,\n\t\t\t\t\tcandidateIdentifierField:String(candidateIdentifierField)\n\t\t\t\t});\n\t\t\t\tif(this.DEBUG)console.log('Extracted identifier from candidate document: '+identifier);\n\t\t\t\tlet selectedCandidate=candidates.find(candidate=>String(candidate[candidateIdentifierField]).toLowerCase()===identifier.toLowerCase());\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\tselectedCandidate=candidates.find(candidate=>String(candidate[candidateIdentifierField]).toLowerCase().includes(identifier.toLowerCase()));\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\tselectedCandidate=candidates.find(candidate=>identifier.toLowerCase().includes(String(candidate[candidateIdentifierField]).toLowerCase()));\n\t\t\t\tif(selectedCandidate)return selectedCandidate;\n\t\t\t\treturn null;\n\t\t\t}))).slice(candidatesForFinalSelection);\n\t\t}\n\t\tif(this.DEBUG)console.log('Selected candidates',selectedCandidates);\n\n\t\treturn{\n\t\t\trationale,\n\t\t\tselectedCandidates\n\t\t};\n\t}\n};\n\nexport namespace AICompareCandidates{\n\texport interface LoadArguments{\n\t\tprogressCallback?:ProgressCallback;\n\t\tmodelName?:string;\n\t};\n\n\texport interface SummarisableSubstringIndices{\n\t\tstart:number;\n\t\tend:number;\n\t};\n\n\texport interface CompareArguments<Candidate>{\n\t\tcandidates:Candidate[];\n\t\tproblemDescription:string;\n\t\tgenerateSearchAreasInstruction?:(problemDescription:string)=>string;\n\t\tconvertCandidateToDocument?:(convertCandidateToDocumentArguments:ConvertCandidateToDocumentArguments<Candidate>)=>string;\n\t\tcandidatesForInitialSelection?:number;\n\t\tcandidatesForFinalSelection?:number;\n\t\tgenerateRankingInstruction?:(generateRankingInstructionArguments:GenerateRankingInstructionArguments)=>string;\n\t\textractIdentifiersFromRationale?:(rationale:string)=>string[];\n\t\textractIdentifierFromCandidateDocument?:(extractIdentifierFromCandidateDocumentArguments:ExtractIdentifierFromCandidateDocumentArguments)=>string;\n\t\tcandidateIdentifierField?:keyof Candidate;\n\t\tgetSummarisableSubstringIndices?:(candidateDocument:string)=>SummarisableSubstringIndices;\n\t};\n\n\texport interface ConvertCandidateToDocumentArguments<Candidate>{\n\t\tcandidate:Candidate;\n\t\tindex:number;\n\t};\n\n\texport interface ExtractIdentifierFromCandidateDocumentArguments{\n\t\tcandidateDocument:string;\n\t\tcandidateIdentifierField:string;\n\t};\n\n\texport interface GenerateRankingInstructionArguments{\n\t\tproblemDescription:string;\n\t\tsummaries:string[];\n\t\tcandidatesForFinalSelection:number;\n\t\tcandidateIdentifierField:string;\n\t};\n\n\texport interface CompareCandidatesReturn<Candidate>{\n\t\tselectedCandidates:Candidate[],\n\t\trationale:string\n\t};\n};\n\nexport default AICompareCandidates;"],"mappings":"2PAiBA,IAAa,EAAb,cAAyC,CAAU,CAClD,IAAa,EACb,MAAM,GAEN,UAAsC,KACtC,mBAAmB,yBACnB,iBAAsD,KACtD,sBAAiD,EAAE,CACnD,0BAAgD,KAChD,yBAAyB,IAAI,gBAE7B,WAAsC,KACtC,oBAAoB,6BACpB,kBAAsD,KACtD,uBAAkD,EAAE,CACpD,2BAAiD,KACjD,0BAA0B,IAAI,gBAE9B,SAAwC,KACxC,kBAAkB,2BAClB,gBAAwD,KACxD,qBAAgD,EAAE,CAClD,yBAA+C,KAC/C,wBAAwB,IAAI,gBAE5B,UAAmC,KACnC,mBAAmB,KAAK,mBACxB,iBAAmD,KACnD,sBAAiD,EAAE,CACnD,0BAAgD,KAChD,yBAAyB,IAAI,gBAE7B,gCAAgC,GAChC,+BAA+B,IAC/B,qCAAqC,IAErC,oBAAoB,GACpB,mBAAmB,IACnB,yBAAyB,IAEzB,iCAAiC,IAEjC,OACC,EAAI,eAAe,GACnB,EAAI,kBAAkB,GACtB,EAAI,iBAAiB,GAGtB,aAAa,CACZ,MAAM,EAAE,CAAC,CAGV,MAAM,cAAc,CACnB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAEzE,GADG,OAAO,GAAY,UAAU,IAAU,KAAK,mBAAmB,GAC/D,CAAC,KAAK,mBAAmB,MAAU,MAAM,+BAA+B,CAc3E,OAbG,IAAiB,KAAK,0BAA0B,GAGnD,KAAK,iBAAiB,EAAS,kBAAkB,KAAK,mBAAmB,CACxE,OAAO,SACP,kBAAkB,IACd,KAAK,OAAM,QAAQ,IAAI,EAAK,UAAU,EAAa,CAAC,CACvD,OAAO,OAAO,KAAK,sBAAsB,EAAa,CAC/C,KAAK,4BAA4B,EAAa,EAEtD,aAAa,KAAK,yBAAyB,OAC3C,CAAC,CACF,KAAK,UAAU,MAAM,KAAK,iBACnB,KAAK,UAGb,MAAM,qBAAqB,CAC1B,mBACA,YAAU,IAC4D,EAAE,CAAC,CAKzE,GAJI,KAAK,kBAAiB,KAAK,cAAc,CAC5C,mBACA,YACA,CAAC,CACC,CAAC,KAAK,UACR,GAAG,CACF,MAAM,KAAK,uBACL,EAAM,CAEZ,KADA,MAAK,iBAAiB,KAChB,EAGR,GAAG,CAAC,KAAK,UAER,KADA,MAAK,iBAAiB,KACZ,MAAM,2BAA2B,CAI7C,MAAM,mBAAmB,EAAY,CACpC,KAAK,yBAAyB,MAAM,EAAO,CAC3C,KAAK,yBAAyB,IAAI,gBAGnC,MAAM,eAAe,CACpB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAEzE,GADG,OAAO,GAAY,UAAU,IAAU,KAAK,oBAAoB,GAChE,CAAC,KAAK,oBAAoB,MAAU,MAAM,gCAAgC,CAc7E,OAbG,IAAiB,KAAK,2BAA2B,GAGpD,KAAK,kBAAkB,EAAS,gBAAgB,KAAK,oBAAoB,CACxE,OAAO,SACP,kBAAkB,IACd,KAAK,OAAM,QAAQ,IAAI,EAAK,UAAU,EAAa,CAAC,CACvD,OAAO,OAAO,KAAK,uBAAuB,EAAa,CAChD,KAAK,6BAA6B,EAAa,EAEvD,aAAa,KAAK,0BAA0B,OAC5C,CAAC,CACF,KAAK,WAAW,MAAM,KAAK,kBACpB,KAAK,WAGb,MAAM,sBAAsB,CAC3B,mBACA,YAAU,IAC4D,EAAE,CAAC,CAKzE,GAJI,KAAK,mBAAkB,KAAK,eAAe,CAC9C,mBACA,YACA,CAAC,CACC,CAAC,KAAK,WACR,GAAG,CACF,MAAM,KAAK,wBACL,EAAM,CAEZ,KADA,MAAK,kBAAkB,KACjB,EAGR,GAAG,CAAC,KAAK,WAER,KADA,MAAK,kBAAkB,KACb,MAAM,4BAA4B,CAI9C,MAAM,qBAAqB,CAC1B,KAAK,0BAA0B,OAAO,CACtC,KAAK,0BAA0B,IAAI,gBAGpC,MAAM,aAAa,CAClB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAEzE,GADG,OAAO,GAAY,UAAU,IAAU,KAAK,kBAAkB,GAC9D,CAAC,KAAK,kBAAkB,MAAU,MAAM,8BAA8B,CAczE,OAbG,IAAiB,KAAK,yBAAyB,GAGlD,KAAK,gBAAgB,EAAS,qBAAqB,KAAK,kBAAkB,CACzE,OAAO,SACP,kBAAkB,IACd,KAAK,OAAM,QAAQ,IAAI,EAAK,UAAU,EAAa,CAAC,CACvD,OAAO,OAAO,KAAK,qBAAqB,EAAa,CAC9C,KAAK,2BAA2B,EAAa,EAErD,aAAa,KAAK,wBAAwB,OAC1C,CAAC,CACF,KAAK,SAAS,MAAM,KAAK,gBAClB,KAAK,SAGb,MAAM,oBAAoB,CACzB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAKzE,GAJI,KAAK,iBAAgB,KAAK,aAAa,CAC1C,mBACA,YACA,CAAC,CACC,CAAC,KAAK,SACR,GAAG,CACF,MAAM,KAAK,sBACL,EAAM,CAEZ,KADA,MAAK,gBAAgB,KACf,EAGR,GAAG,CAAC,KAAK,SAER,KADA,MAAK,gBAAgB,KACX,MAAM,0BAA0B,CAI5C,MAAM,mBAAmB,CACxB,KAAK,wBAAwB,OAAO,CACpC,KAAK,wBAAwB,IAAI,gBAGlC,MAAM,cAAc,CACnB,mBACA,YAAU,IAC4D,EAAE,CAAC,CAEzE,GADG,OAAO,GAAY,UAAU,IAAU,KAAK,mBAAmB,GAC/D,CAAC,KAAK,mBAAmB,MAAU,MAAM,+BAA+B,CAa3E,OAZG,IAAiB,KAAK,0BAA0B,GAGnD,KAAK,iBAAiB,EAAc,gBAAgB,KAAK,mBAAmB,CAC3E,kBAAkB,IACd,KAAK,OAAM,QAAQ,IAAI,EAAK,UAAU,EAAa,CAAC,CACvD,OAAO,OAAO,KAAK,sBAAsB,EAAa,CAC/C,KAAK,4BAA4B,EAAa,EAEtD,aAAa,KAAK,yBAAyB,OAC3C,CAAC,CACF,KAAK,UAAU,MAAM,KAAK,iBACnB,KAAK,UAGb,MAAM,qBAAqB,CAC1B,mBACA,YAAU,IAC4D,EAAE,CAAC,CAKzE,GAJI,KAAK,kBAAiB,KAAK,cAAc,CAC5C,mBACA,YACA,CAAC,CACC,CAAC,KAAK,UACR,GAAG,CACF,MAAM,KAAK,uBACL,EAAM,CAEZ,KADA,MAAK,iBAAiB,KAChB,EAGR,GAAG,CAAC,KAAK,UAER,KADA,MAAK,iBAAiB,KACZ,MAAM,2BAA2B,CAI7C,MAAM,oBAAoB,CACzB,KAAK,yBAAyB,OAAO,CACrC,KAAK,yBAAyB,IAAI,gBAGnC,MAAM,WAAW,EAA8B,CAE9C,OADA,MAAM,KAAK,qBAAqB,CACzB,MAAM,MAAM,MAAM,KAAK,WAAW,EAAK,CAC7C,QAAQ,OACR,UAAU,GACV,CAAC,GAAG,KAAK,CAGX,MAAM,eAAe,EAAmC,CACvD,OAAO,QAAQ,IAAI,EAAM,IAAI,GAAM,KAAK,WAAW,EAAK,CAAC,CAAC,CAG3D,uBAAuB,EAAc,CACpC,MAAO;;;EAEN,EACA;;eAGF,sCAAsC,EAA0B,CAC/D,MAAO;WAAyG,EAAmB,IAGpI,kCAA6C,CAC5C,YACA,SACwI,EAAE,CAAC,CAC3I,IAAI,EAAS,uBAAuB,EACpC,IAAI,IAAI,KAAK,EAAU,GAAU;EAAK,EAAO,UAAU,EAAE,CAAC,MAAM,OAAO,EAAU,IAAK,SAAS,EAAK,UAAU,EAAU,GAAG,CAAC,OAAO,EAAU,GAAG,EAEhJ,MADA,IAAU;oBAAuB,EAC1B,EAGR,kCAAkC,CACjC,qBACA,YACA,8BACA,4BACkH,EAAE,CAAC,CACrH,MAAO;uBACkB,EAA4B;mCAChB,EAAmB,QAAQ,WAAW,IAAI,CAAC;;;;;EAE7D,EAAU,KAAK;;EAAO,CAAC;;;YAE5B,EAAO,UAAU,EAAyB,CAAC;SAI1D,aAAa,EAAY,EAAa,EAAkB,CACpD,IAAI,EAAc,EAAK,MAAM,EAAW,CAAC,OAAO,EAAM,CACtD,OAAO,EAAc,EAAE,EAAc,EAAc,EAGvD,8CAA8C,CAC7C,oBACA,4BAC0I,EAAE,CAAC,CAC1I,KAAK,OAAM,QAAQ,IAAI,EAAkB,EAAyB,CACrE,IAAI,EAAU,EAAO,UAAU,EAAyB,CACpD,EAAW,EAAkB,QAAQ,EAAU,CAQnD,GAPG,EAAW,IAAE,EAAW,EAAkB,aAAa,CAAC,QAAQ,EAAU,aAAa,CAAC,EACxF,KAAK,OAAM,QAAQ,IAAI,EAAW,CAClC,GAAY,IAAE,GAAY,EAAU,QACpC,EAAW,IACb,EAAW,EAAkB,aAAa,CAAC,QAAQ,EAAyB,aAAa,CAAC,CACvF,GAAY,IAAE,GAAY,EAAyB,SAEpD,KAAK,MAAM,QAAQ,IAAI,EAAW,MAChC,MAAO,GAKZ,GAJA,EAAW,EAAkB,QAAQ,IAAI,EAAW,CACjD,KAAK,OAAM,QAAQ,IAAI,EAAW,CAClC,EAAW,IAAE,EAAW,KAAK,aAAa,EAAkB,MAAM,EAAW,EAC7E,KAAK,OAAM,QAAQ,IAAI,EAAW,CAClC,EAAW,EAAE,MAAO,GACvB,IAAI,EAAS,EAAkB,QAAQ;EAAK,EAAW,CAGvD,OAFG,EAAS,IAAE,EAAS,EAAkB,QACtC,KAAK,OAAM,QAAQ,IAAI,EAAS,CAC5B,EAAkB,UAAU,EAAW,EAAS,CAAC,MAAM,CAG/D,uCAAuC,EAAiB,CACvD,IAAI,EAAM,kCACNA,EAAiB,EAAE,CACvB,IAAI,IAAIC,EAA2B,MAAM,QAAQ,EAAM,EAAM,KAAK,EAAU,CAAC,EAAK,EAAM,IAAG,EAAQ,KAAK,EAAM,GAAG,CACjH,OAAO,EAGR,MAAM,kBAA6B,CAClC,aACA,qBAAmB,GACnB,iCAA+B,KAAK,sCAAsC,KAAK,KAAK,CACpF,6BAA2B,KAAK,kCAAkC,KAAK,KAAK,CAC5E,gCAA8B,EAC9B,8BAA4B,EAC5B,6BAA2B,KAAK,kCAAkC,KAAK,KAAK,CAC5E,kCAAgC,KAAK,uCAAuC,KAAK,KAAK,CACtF,yCAAuC,KAAK,8CAA8C,KAAK,KAAK,CACpG,2BAAyB,IAAA,GACzB,mCACkG,EAAE,CAAsE,CAC1K,GAAG,CAAC,MAAM,QAAQ,EAAW,EAAE,EAAW,QAAQ,EAAE,MAAU,MAAM,yBAAyB,CAI7F,GAHA,EAA8B,EAAO,cAAc,EAA8B,CAC9E,GAA+B,IAClC,EAA4B,EAAO,cAAc,EAA4B,CAC1E,GAA6B,GAAE,MAAU,MAAM,4EAA4E,CAC9H,GAAG,EAA8B,EAA4B,MAAU,MAAM,6FAA6F,CAC1K,GAAG,EAA8B,EAAW,OAAO,MAAU,MAAM,aAAa,EAA8B,yFAAyF,EAAW,OAAO,CACzN,GAAG,EAA4B,EAAW,OAAO,MAAU,MAAM,aAAa,EAA4B,yFAAyF,EAAW,OAAO,CACrN,GAAG,CAAC,IACH,EAAyB,OAAO,KAAK,EAAW,GAAa,CAAC,GAC3D,CAAC,GAAyB,MAAU,MAAM,gCAAgC,CAG9E,IAAI,EAAU,GACVC,EAA+B,EAAE,CAGrC,GADA,MAAM,KAAK,qBAAqB,CAC7B,CAAC,KAAK,SAAS,OAClB,IAAI,EAAmB,EAAW,KAAK,EAAU,IAAQ,EAA2B,CACnF,YACA,QACA,CAAC,CAAC,CACC,EAAY,MAAM,EAAkB,UACvC,EAAO,UAAU,EAAmB,CACpC,EAAmB,KAAK,EAAS,IAAQ,EAAM,CAC/C,KACA,CAEG,EAA0B,KAAK,uBAAuB,EAA+B,EAAmB,CAAC,CAG7G,GAFG,KAAK,OAAM,QAAQ,IAAI,kCAAkC,EAA0B,CACtF,MAAM,KAAK,sBAAsB,CAC9B,CAAC,KAAK,UAAU,OAEnB,GAD4B,KAAK,UAAU,OAAO,EAA0B,CACjD,OAAO,KAAK,UAAU,iBAAiB,MAAU,MAAM,sEAAsE,CAGxJ,GADA,MAAM,KAAK,sBAAsB,CAC9B,CAAC,KAAK,UAAU,OACnB,IAAI,EAAa,KAAK,UAAU,cAAc,KAAK,UAAU,cAAc,EACvE,EAAa,KAAK,UAAU,cAAc,EAC1C,EAAsB,MAAM,KAAK,UAAU,EAA0B,CACxE,eAAe,KAAK,gCACpB,YAAY,KAAK,+BACjB,mBAAmB,KAAK,qCACxB,eACA,eACA,CAAC,CACE,EAAiB,MAAM,QAAQ,IAAwB,GAAG,CAAC,IAAwB,KAAK,GAAG,IAAwB,GACvH,GAAG,CAAC,EAAiB,eAAe,MAAU,MAAM,qCAAqC,CACtF,KAAK,OAAM,QAAQ,IAAI,oCAAoC,EAAiB,eAAe,CAC9F,IAAI,EAAyB,EAAiB,eAAe,UAAU,CAAC,QAAQ,gBAAgB,CAC7F,GAA0B,EAAE,GAA0B,GACpD,EAAyB,EAE9B,IAAI,EAAkB,EAAiB,eAAe,UAAU,CAAC,UAAU,EAAyB,CAAC,MAAM,CAGxG,KAAK,OAAM,QAAQ,IAAI,wBAAwB,EAAkB,CACpE,IAAI,EAAY,MAAM,EAAY,iBAAiB,EAAkB,EAA8B,CAChG,KAAK,OAAM,QAAQ,IAAI,0BAA0B,EAAY,CAEhE,IAAIC,EAAmB,EAAE,CAEzB,GAAG,EAAY,KAAK,GAAQ,EAAO,YAAY,MAAM,CAAC,MAAM,MAAM,CAAC,OAAO,KAAK,iCAAiC,CAAC,CAEhH,GADA,MAAM,KAAK,uBAAuB,CAC/B,CAAC,KAAK,WAAW,OACpB,GAAW,MAAM,QAAQ,WAAW,EAAY,IAAI,KAAM,IAAQ,CACjE,GAAG,CAAC,EAAO,aAAa,OAAO,EAAO,aAAc,SAAS,MAAO,GACpE,GAAG,EAAO,YAAY,MAAM,CAAC,MAAM,MAAM,CAAC,QAAQ,KAAK,iCAAiC,OAAO,EAAO,YACtG,IAAIC,EAA8E,CACjF,MAAM,EACN,IAAI,EAAO,YAAY,OACvB,CACE,GAAgC,OAAO,OAAO,EAA6B,EAAgC,EAAO,YAAY,CAAC,CAClI,EAA6B,MAAM,EAAO,MAAM,EAAO,cAAc,EAA6B,MAAM,CAAC,EAAE,EAAO,YAAY,OAAO,CACrI,EAA6B,IAAI,EAAO,MAAM,EAAO,cAAc,EAA6B,IAAI,CAAC,EAAE,EAAO,YAAY,OAAO,CACjI,IAAI,EAAsB,EAAO,YAAY,UAAU,EAA6B,MAAM,EAA6B,IAAI,CACvH,EAAc,EAAO,YAAY,UAAU,EAAE,EAA6B,MAAM,CAChF,EAAa,EAAO,YAAY,UAAU,EAA6B,IAAI,CAC3E,EAAyB,EAAc,MAAM,KAAK,CAAC,OAAO,EAAa,MAAM,KAAK,CAAC,OACnF,EAAoC,KAAK,IAAI,EAAE,IAAI,EAAyB,CAC5E,EAAyB,MAAM,KAAK,aAAa,EAA4C,CAChG,WAAW,EACX,CAAC,CAEE,EAAiB,IADG,MAAM,QAAQ,IAA2B,GAAG,CAAC,IAA2B,KAAK,GAAG,IAA2B,KAC1E,cAAc,IAAI,MAAM,KAAK,CAAC,MAAM,EAAoC,CAAC,KAAK,IAAI,CAAC,EAE5I,OADG,KAAK,OAAM,QAAQ,IAAI,yBAAyB,EAAiB,CAC7D,GACN,CAAC,EAAE,OAAO,GAAQ,EAAO,SAAS,aAAa,EAAO,MAAM,CAAC,IAAI,GAAS,EAA0C,MAAM,MAE5H,EAAU,EAAY,IAAI,GAAQ,EAAO,YAAY,CAGtD,IAAI,EAAsB,KAAK,uBAAuB,EAA2B,CAChF,qBACA,YACA,8BACA,yBAAyB,OAAO,EAAyB,CACzD,CAAC,CAAC,CAGH,GAFG,KAAK,OAAM,QAAQ,IAAI,6BAA6B,EAAsB,CACrD,KAAK,UAAU,OAAO,EAAsB,CAC7C,OAAO,KAAK,UAAU,iBAAiB,MAAU,MAAM,iEAAiE,CAC/I,IAAI,EAAa,MAAM,KAAK,UAAU,EAAsB,CAC3D,eAAe,KAAK,oBACpB,YAAY,KAAK,mBACjB,mBAAmB,KAAK,yBACxB,eACA,eACA,CAAC,CAEF,GADY,MAAM,QAAQ,IAAe,GAAG,CAAC,IAAe,KAAK,GAAG,EAAa,IAC/D,eAAe,UAAU,CAAC,MAAM,CAAC,QAAQ,gCAAiC,GAAG,CAC5F,KAAK,OAAM,QAAQ,IAAI,wBAAwB,EAAU,CAC5D,IAAI,EAAuB,EAAU,QAAQ,gBAAgB,CAM7D,GALG,GAAwB,EAAE,GAAwB,GAChD,EAAuB,EAC5B,EAAU,EAAU,UAAU,EAAuB,CAGlD,EAAU,CACZ,IAAI,EAAY,EAAgC,EAAU,CACvD,EAAY,OAAO,IAA4B,EAAY,EAAY,MAAM,EAAE,EAA4B,EAC9G,EAAmB,EAAO,QAAQ,EAAY,IAAI,GAAY,CAC7D,IAAI,EAAkB,EAAW,KAAK,GAAW,OAAO,EAAU,GAA0B,CAAC,aAAa,GAAG,EAAW,aAAa,CAAC,CAMtI,OALG,IACH,EAAkB,EAAW,KAAK,GAAW,OAAO,EAAU,GAA0B,CAAC,aAAa,CAAC,SAAS,EAAW,aAAa,CAAC,CAAC,CACvI,KACH,EAAkB,EAAW,KAAK,GAAW,EAAW,aAAa,CAAC,SAAS,OAAO,EAAU,GAA0B,CAAC,aAAa,CAAC,CAAC,CACvI,GAAyB,EACrB,MACN,CAAC,CAqBJ,OAlBG,CAAC,MAAM,QAAQ,EAAmB,EAAE,EAAmB,QAAQ,KACjE,EAAmB,EAAO,KAAK,EAAO,QAAQ,EAAY,IAAI,GAAQ,CACrE,IAAI,EAAW,EAAuC,CACrD,kBAAkB,EAAO,YACzB,yBAAyB,OAAO,EAAyB,CACzD,CAAC,CACC,KAAK,OAAM,QAAQ,IAAI,iDAAiD,EAAW,CACtF,IAAI,EAAkB,EAAW,KAAK,GAAW,OAAO,EAAU,GAA0B,CAAC,aAAa,GAAG,EAAW,aAAa,CAAC,CAMtI,OALG,IACH,EAAkB,EAAW,KAAK,GAAW,OAAO,EAAU,GAA0B,CAAC,aAAa,CAAC,SAAS,EAAW,aAAa,CAAC,CAAC,CACvI,KACH,EAAkB,EAAW,KAAK,GAAW,EAAW,aAAa,CAAC,SAAS,OAAO,EAAU,GAA0B,CAAC,aAAa,CAAC,CAAC,CACvI,GAAyB,EACrB,MACN,CAAC,CAAC,CAAC,MAAM,EAA4B,EAErC,KAAK,OAAM,QAAQ,IAAI,sBAAsB,EAAmB,CAE7D,CACL,YACA,qBACA,2BAoDH,IAAA,EAAe"}

package/example/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ai-compare-candidates-example",
-	"version": "0.0.2",
+	"version": "0.0.4",
 	"description": "An example for the ai-compare-candidates library",
 	"productName": "AI Compare Candidates Example",
 	"author": "Wilson Foo Yu Kang <wilson.foo@customautosys.com>",

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ai-compare-candidates",
-	"version": "0.0.2",
+	"version": "0.0.4",
 	"description": "Compare and rank multiple candidate objects using artificial intelligence retrieval augmented generation, providing the rationale",
 	"main": "dist/index.mjs",
 	"scripts": {
@@ -27,9 +27,9 @@
 	"homepage": "https://github.com/customautosys/ai-compare-candidates#readme",
 	"packageManager": "yarn@4.12.0",
 	"dependencies": {
-		"@huggingface/transformers": "^3.8.1",
 		"@langchain/classic": "^1.0.6",
 		"@langchain/core": "^1.1.5",
+		"@sroussey/transformers": "^3.8.2",
 		"jsan": "^3.1.14",
 		"lodash": "^4.17.21"
 	},

package/src/index.ts

@@ -9,7 +9,7 @@ import{
 	FeatureExtractionPipeline,
 	PreTrainedTokenizer,
 	TextGenerationConfig
-}from '@huggingface/transformers';
+}from '@sroussey/transformers';
 import {MemoryVectorStore} from '@langchain/classic/vectorstores/memory';
 import {Embeddings} from '@langchain/core/embeddings';
 import lodash from 'lodash';
@@ -24,24 +24,28 @@ export class AICompareCandidates extends Embeddings{
 	generatorPromise:Promise<TextGenerationPipeline>|null=null;
 	generatorProgressInfo:ProgressInfo=<ProgressInfo>{};
 	generatorProgressCallback:ProgressCallback|null=null;
+	generatorAbortController=new AbortController();
 
 	summariser:SummarizationPipeline|null=null;
 	summariserModelName='Xenova/distilbart-cnn-12-6';
 	summariserPromise:Promise<SummarizationPipeline>|null=null;
 	summariserProgressInfo:ProgressInfo=<ProgressInfo>{};
 	summariserProgressCallback:ProgressCallback|null=null;
+	summariserAbortController=new AbortController();
 
 	embedder:FeatureExtractionPipeline|null=null;
 	embedderModelName='Xenova/all-MiniLM-L12-v2';
 	embedderPromise:Promise<FeatureExtractionPipeline>|null=null;
 	embedderProgressInfo:ProgressInfo=<ProgressInfo>{};
 	embedderProgressCallback:ProgressCallback|null=null;
+	embedderAbortController=new AbortController();
 
 	tokeniser:PreTrainedTokenizer|null=null;
 	tokeniserModelName=this.generatorModelName;
 	tokeniserPromise:Promise<PreTrainedTokenizer>|null=null;
 	tokeniserProgressInfo:ProgressInfo=<ProgressInfo>{};
 	tokeniserProgressCallback:ProgressCallback|null=null;
+	tokeniserAbortController=new AbortController();
 
 	generateSearchAreasMaxNewTokens=64;
 	generateSearchAreasTemperature=0.35;
@@ -78,16 +82,38 @@ export class AICompareCandidates extends Embeddings{
 				if(this.DEBUG)console.log(jsan.stringify(progressInfo));
 				Object.assign(this.generatorProgressInfo,progressInfo);
 				return this.generatorProgressCallback?.(progressInfo);
-			}
+			},
+			abort_signal:this.generatorAbortController.signal
 		});
 		this.generator=await this.generatorPromise;
 		return this.generator;
 	}
 
-	async checkGeneratorLoaded(){
-		if(!this.generatorPromise)this.loadGenerator();
-		if(!this.generator)await this.generatorPromise;
-		if(!this.generator)throw new Error('Unable to load generator');
+	async checkGeneratorLoaded({
+		progressCallback,
+		modelName=''
+	}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){
+		if(!this.generatorPromise)this.loadGenerator({
+			progressCallback,
+			modelName
+		});
+		if(!this.generator){
+			try{
+				await this.generatorPromise;
+			}catch(error){
+				this.generatorPromise=null;
+				throw error;
+			}
+		}
+		if(!this.generator){
+			this.generatorPromise=null;
+			throw new Error('Unable to load generator');
+		}
+	}
+
+	async abortLoadGenerator(reason?:any){
+		this.generatorAbortController.abort(reason);
+		this.generatorAbortController=new AbortController();
 	}
 
 	async loadSummariser({
@@ -105,16 +131,38 @@ export class AICompareCandidates extends Embeddings{
 				if(this.DEBUG)console.log(jsan.stringify(progressInfo));
 				Object.assign(this.summariserProgressInfo,progressInfo);
 				return this.summariserProgressCallback?.(progressInfo);
-			}
+			},
+			abort_signal:this.summariserAbortController.signal
 		});
 		this.summariser=await this.summariserPromise;
 		return this.summariser;
 	}
 
-	async checkSummariserLoaded(){
-		if(!this.summariserPromise)this.loadSummariser();
-		if(!this.summariser)await this.summariserPromise;
-		if(!this.summariser)throw new Error('Unable to load summariser');
+	async checkSummariserLoaded({
+		progressCallback,
+		modelName=''
+	}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){
+		if(!this.summariserPromise)this.loadSummariser({
+			progressCallback,
+			modelName
+		});
+		if(!this.summariser){
+			try{
+				await this.summariserPromise;
+			}catch(error){
+				this.summariserPromise=null;
+				throw error;
+			}
+		}
+		if(!this.summariser){
+			this.summariserPromise=null;
+			throw new Error('Unable to load summariser');
+		}
+	}
+
+	async abortLoadSummariser(){
+		this.summariserAbortController.abort();
+		this.summariserAbortController=new AbortController();
 	}
 
 	async loadEmbedder({
@@ -132,16 +180,38 @@ export class AICompareCandidates extends Embeddings{
 				if(this.DEBUG)console.log(jsan.stringify(progressInfo));
 				Object.assign(this.embedderProgressInfo,progressInfo);
 				return this.embedderProgressCallback?.(progressInfo);
-			}
+			},
+			abort_signal:this.embedderAbortController.signal
 		});
 		this.embedder=await this.embedderPromise;
 		return this.embedder;
 	}
 
-	async checkEmbedderLoaded(){
-		if(!this.embedderPromise)this.loadEmbedder();
-		if(!this.embedder)await this.embedderPromise;
-		if(!this.embedder)throw new Error('Unable to load embedder');
+	async checkEmbedderLoaded({
+		progressCallback,
+		modelName=''
+	}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){
+		if(!this.embedderPromise)this.loadEmbedder({
+			progressCallback,
+			modelName
+		});
+		if(!this.embedder){
+			try{
+				await this.embedderPromise;
+			}catch(error){
+				this.embedderPromise=null;
+				throw error;
+			}
+		}
+		if(!this.embedder){
+			this.embedderPromise=null;
+			throw new Error('Unable to load embedder');
+		}
+	}
+
+	async abortLoadEmbedder(){
+		this.embedderAbortController.abort();
+		this.embedderAbortController=new AbortController();
 	}
 
 	async loadTokeniser({
@@ -158,16 +228,38 @@ export class AICompareCandidates extends Embeddings{
 				if(this.DEBUG)console.log(jsan.stringify(progressInfo));
 				Object.assign(this.tokeniserProgressInfo,progressInfo);
 				return this.tokeniserProgressCallback?.(progressInfo);
-			}
-		})
+			},
+			abort_signal:this.tokeniserAbortController.signal
+		});
 		this.tokeniser=await this.tokeniserPromise;
 		return this.tokeniser;
 	}
 
-	async checkTokeniserLoaded(){
-		if(!this.tokeniserPromise)this.loadTokeniser();
-		if(!this.tokeniser)await this.tokeniserPromise;
-		if(!this.tokeniser)throw new Error('Unable to load tokeniser');
+	async checkTokeniserLoaded({
+		progressCallback,
+		modelName=''
+	}:AICompareCandidates.LoadArguments=<AICompareCandidates.LoadArguments>{}){
+		if(!this.tokeniserPromise)this.loadTokeniser({
+			progressCallback,
+			modelName
+		});
+		if(!this.tokeniser){
+			try{
+				await this.tokeniserPromise;
+			}catch(error){
+				this.tokeniserPromise=null;
+				throw error;
+			}
+		}
+		if(!this.tokeniser){
+			this.tokeniserPromise=null;
+			throw new Error('Unable to load tokeniser');
+		}
+	}
+
+	async abortLoadTokeniser(){
+		this.tokeniserAbortController.abort();
+		this.tokeniserAbortController=new AbortController();
 	}
 
 	async embedQuery(text:string):Promise<number[]>{