@ax-llm/ax 9.0.20 → 9.0.22

package/docs/manager.d.ts

@@ -0,0 +1,45 @@
+import { type AxAIService, type AxDBService, type AxProgram } from '../index.js';
+export type AxRewriteIn = {
+    query: string;
+};
+export type AxRewriteOut = {
+    rewrittenQuery: string;
+};
+export type AxRerankerIn = {
+    query: string;
+    items: string[];
+};
+export type AxRerankerOut = {
+    rankedItems: string[];
+};
+export interface AxDBLoaderOptions {
+    chunker?: (text: string) => string[];
+    rewriter?: AxProgram<AxRewriteIn, AxRewriteOut>;
+    reranker?: AxProgram<AxRerankerIn, AxRerankerOut>;
+}
+export interface AxDBManagerArgs {
+    ai: AxAIService;
+    db: AxDBService;
+    config?: AxDBLoaderOptions;
+}
+export interface AxDBMatch {
+    score: number;
+    text: string;
+}
+export declare class AxDBManager {
+    private ai;
+    private db;
+    private chunker;
+    private rewriter?;
+    private reranker?;
+    constructor({ ai, db, config }: Readonly<AxDBManagerArgs>);
+    private defaultChunker;
+    insert: (text: Readonly<string | string[]>, options?: Readonly<{
+        batchSize?: number;
+        maxWordsPerChunk?: number;
+        minWordsPerChunk?: number;
+    }>) => Promise<void>;
+    query: (query: Readonly<string | string[] | number | number[]>, { topPercent }?: Readonly<{
+        topPercent?: number;
+    }> | undefined) => Promise<AxDBMatch[][]>;
+}

package/docs/manager.js

@@ -0,0 +1,159 @@
+import {} from '../index.js';
+const table = '_internal';
+export class AxDBManager {
+    ai;
+    db;
+    chunker;
+    rewriter;
+    reranker;
+    constructor({ ai, db, config }) {
+        this.ai = ai;
+        this.db = db;
+        this.chunker = config?.chunker ?? this.defaultChunker;
+        this.reranker = config?.reranker;
+        this.rewriter = config?.rewriter;
+    }
+    defaultChunker = (text) => {
+        // Default chunking by paragraphs
+        return text.split(/\n\n+/);
+    };
+    insert = async (text, options) => {
+        try {
+            const chunkerInput = Array.isArray(text)
+                ? text.join('\n\n')
+                : text;
+            // Chunk the text using the specified or default chunking function
+            const initialChunks = this.chunker(chunkerInput).filter((chunk) => chunk.length > 0);
+            const maxWordsPerChunk = options?.maxWordsPerChunk;
+            const minWordsPerChunk = options?.minWordsPerChunk;
+            const chunks = processChunks({
+                initialChunks,
+                minWordsPerChunk,
+                maxWordsPerChunk
+            });
+            const bs = options?.batchSize ?? 10;
+            // Process chunks in batches of 10
+            for (let i = 0; i < chunks.length; i += bs) {
+                const batch = chunks.slice(i, i + bs);
+                // Get embeddings for the whole batch from the AI service in one call
+                const ret = await this.ai.embed({ texts: batch });
+                // Prepare batch for bulk upsert
+                const embeddings = ret.embeddings
+                    .map((embedding, index) => ({
+                    id: `chunk_${Date.now() + index}`, // Unique ID for each chunk, adjusted by index
+                    table,
+                    values: embedding,
+                    metadata: { text: batch[index] ?? '' }
+                }))
+                    .filter((v) => v.metadata?.text && v.metadata?.text.length > 0);
+                // Batch upsert embeddings
+                await this.db.batchUpsert(embeddings);
+            }
+        }
+        catch (error) {
+            throw new Error(`Error processing text: ${error}`);
+        }
+    };
+    query = async (query, { topPercent } = {}) => {
+        const texts = Array.isArray(query) ? query : [query];
+        if (typeof texts[0] === 'string' && this.rewriter) {
+            for (const [i, text] of texts.entries()) {
+                const { rewrittenQuery } = await this.rewriter.forward({
+                    query: text
+                });
+                texts[i] = rewrittenQuery;
+            }
+        }
+        let queries;
+        if (typeof texts[0] === 'string') {
+            const embedResults = await this.ai.embed({ texts });
+            queries = embedResults.embeddings.map((values) => this.db.query({ table, values }));
+        }
+        else {
+            queries = texts.map((values) => this.db.query({ table, values }));
+        }
+        const queryResults = await Promise.all(queries);
+        const res = [];
+        for (const { matches } of queryResults) {
+            const m = matches
+                .filter((v) => v.metadata?.text && v.metadata?.text.length > 0)
+                .map(({ score, metadata }) => ({ score, text: metadata?.text ?? '' }));
+            const tp = topPercent && topPercent > 1 ? topPercent / 100 : topPercent;
+            const resultItems = tp ? getTopInPercent(m, tp) : m;
+            if (this.reranker) {
+                const { rankedItems } = await this.reranker.forward({
+                    query: texts[0],
+                    items: resultItems.map((item) => item.text)
+                });
+                const items = rankedItems
+                    .map((item) => resultItems.find((r) => r.text === item))
+                    .filter((v) => v !== undefined);
+                res.push(items);
+            }
+            else {
+                res.push(resultItems);
+            }
+        }
+        return res;
+    };
+}
+const processChunks = ({ initialChunks, maxWordsPerChunk = 350, minWordsPerChunk = 250 }) => {
+    const chunks = [];
+    let currentChunk = '';
+    let currentWordCount = 0;
+    initialChunks.forEach((chunk) => {
+        const words = chunk.split(/\s+/); // Split the chunk into words
+        const wordCount = words.length; // Count words in the current chunk
+        if (currentWordCount + wordCount <= maxWordsPerChunk) {
+            // Add to the current chunk if within the max size limit
+            currentChunk += chunk + '\n\n';
+            currentWordCount += wordCount;
+        }
+        else if (currentWordCount > 0 &&
+            currentWordCount + wordCount <= maxWordsPerChunk * 1.5) {
+            // If the total word count exceeds the limit but is less than 150% of the maxWordsPerChunk
+            currentChunk += chunk + '\n\n';
+            currentWordCount += wordCount;
+        }
+        else {
+            // If the current chunk is not empty and adding the new chunk exceeds the adjusted limit
+            if (currentWordCount > minWordsPerChunk) {
+                chunks.push(currentChunk.trim());
+                currentChunk = '';
+                currentWordCount = 0;
+            }
+            // Handle the case where the chunk itself is larger than the limit
+            if (wordCount > maxWordsPerChunk) {
+                const remainingWords = words;
+                while (remainingWords.length > maxWordsPerChunk * 1.5) {
+                    const slice = remainingWords.splice(0, maxWordsPerChunk);
+                    chunks.push(slice.join(' '));
+                }
+                // Add the last portion if it fits the condition of being within 150% of maxWordsPerChunk
+                if (remainingWords.length > 0) {
+                    currentChunk += remainingWords.join(' ') + '\n\n';
+                    currentWordCount += remainingWords.length;
+                }
+            }
+            else {
+                // If the new chunk is smaller than the maximum words per chunk
+                currentChunk = chunk + '\n\n';
+                currentWordCount = wordCount;
+            }
+        }
+    });
+    // Push the last chunk if it exists and meets the minimum words condition
+    if (currentWordCount > minWordsPerChunk || chunks.length === 0) {
+        chunks.push(currentChunk.trim());
+    }
+    return chunks;
+};
+const getTopInPercent = (entries, percent = 0.1) => {
+    // Sort entries by score in ascending order
+    const sortedEntries = [...entries].sort((a, b) => a.score - b.score);
+    // Calculate the number of entries to take (top 10%)
+    const topTenPercentCount = Math.ceil(sortedEntries.length * percent);
+    // Return the top 10% of entries
+    return sortedEntries.slice(0, topTenPercentCount);
+};
+//# sourceMappingURL=manager.js.map

package/docs/manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../docs/manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAKN,MAAM,aAAa,CAAC;AAyBrB,MAAM,KAAK,GAAG,WAAW,CAAC;AAE1B,MAAM,OAAO,WAAW;IACd,EAAE,CAAc;IAChB,EAAE,CAAc;IAChB,OAAO,CAA6B;IACpC,QAAQ,CAAwC;IAChD,QAAQ,CAA0C;IAE1D,YAAY,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,EAA6B;QACvD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,QAAQ,GAAG,MAAM,EAAE,QAAQ,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,MAAM,EAAE,QAAQ,CAAC;IACnC,CAAC;IAEO,cAAc,GAAG,CAAC,IAAY,EAAY,EAAE;QAClD,iCAAiC;QACjC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,GAAG,KAAK,EACZ,IAAiC,EACjC,OAIE,EACa,EAAE;QACjB,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBACtC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;gBACnB,CAAC,CAAE,IAAe,CAAC;YAErB,kEAAkE;YAClE,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,CACrD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAC5B,CAAC;YAEF,MAAM,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,CAAC;YACnD,MAAM,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,CAAC;YAEnD,MAAM,MAAM,GAAG,aAAa,CAAC;gBAC3B,aAAa;gBACb,gBAAgB;gBAChB,gBAAgB;aACjB,CAAC,CAAC;YAEH,MAAM,EAAE,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;YAEpC,kCAAkC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;gBAEtC,qEAAqE;gBACrE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;gBAElD,gCAAgC;gBAChC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU;qBAC9B,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1B,EAAE,EAAE,SAAS,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,EAAE,8CAA8C;oBACjF,KAAK;oBACL,MAAM,EAAE,SAAS;oBACjB,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;iBACvC,CAAC,CAAC;qBACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAElE,0BAA0B;gBAC1B,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,GAAG,KAAK,EACX,KAAsD,EACtD,EAAE,UAAU,KAAoD,EAAE,EAC1C,EAAE;QAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAErD,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClD,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBACxC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;oBACrD,KAAK,EAAE,IAAI;iBACZ,CAAC,CAAC;gBACH,KAAK,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,OAAqC,CAAC;QAE1C,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YACpD,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAC/C,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CACjC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,GAAG,GAAkB,EAAE,CAAC;QAE9B,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,YAAY,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,OAAO;iBACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;iBAC9D,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;YAEzE,MAAM,EAAE,GAAG,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;YACxE,MAAM,WAAW,GAAG,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEpD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAClD,KAAK,EAAE,KAAK,CAAC,CAAC,CAAW;oBACzB,KAAK,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;iBAC5C,CAAC,CAAC;gBAEH,MAAM,KAAK,GAAG,WAAW;qBACtB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;qBACvD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAgB,CAAC;gBAEjD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;CACH;AAED,MAAM,aAAa,GAAG,CAAC,EACrB,aAAa,EACb,gBAAgB,GAAG,GAAG,EACtB,gBAAgB,GAAG,GAAG,EAKtB,EAAY,EAAE;IACd,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,aAAa,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,6BAA6B;QAC/D,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,mCAAmC;QAEnE,IAAI,gBAAgB,GAAG,SAAS,IAAI,gBAAgB,EAAE,CAAC;YACrD,wDAAwD;YACxD,YAAY,IAAI,KAAK,GAAG,MAAM,CAAC;YAC/B,gBAAgB,IAAI,SAAS,CAAC;QAChC,CAAC;aAAM,IACL,gBAAgB,GAAG,CAAC;YACpB,gBAAgB,GAAG,SAAS,IAAI,gBAAgB,GAAG,GAAG,EACtD,CAAC;YACD,0FAA0F;YAC1F,YAAY,IAAI,KAAK,GAAG,MAAM,CAAC;YAC/B,gBAAgB,IAAI,SAAS,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,wFAAwF;YACxF,IAAI,gBAAgB,GAAG,gBAAgB,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjC,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,CAAC;YACvB,CAAC;YACD,kEAAkE;YAClE,IAAI,SAAS,GAAG,gBAAgB,EAAE,CAAC;gBACjC,MAAM,cAAc,GAAG,KAAK,CAAC;gBAC7B,OAAO,cAAc,CAAC,MAAM,GAAG,gBAAgB,GAAG,GAAG,EAAE,CAAC;oBACtD,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;oBACzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/B,CAAC;gBACD,yFAAyF;gBACzF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,YAAY,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;oBAClD,gBAAgB,IAAI,cAAc,CAAC,MAAM,CAAC;gBAC5C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,+DAA+D;gBAC/D,YAAY,GAAG,KAAK,GAAG,MAAM,CAAC;gBAC9B,gBAAgB,GAAG,SAAS,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,IAAI,gBAAgB,GAAG,gBAAgB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/D,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CACtB,OAA6B,EAC7B,UAAkB,GAAG,EACR,EAAE;IACf,2CAA2C;IAC3C,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAErE,oDAAoD;IACpD,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC;IAErE,gCAAgC;IAChC,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;AACpD,CAAC,CAAC"}

package/docs/reranker.d.ts

@@ -0,0 +1,7 @@
+import type { AxAIService } from '../ai/types.js';
+import { AxGenerate, type AxGenerateOptions, type AxProgramForwardOptions } from '../dsp/index.js';
+import type { AxRerankerIn, AxRerankerOut } from './manager.js';
+export declare class AxDefaultResultReranker extends AxGenerate<AxRerankerIn, AxRerankerOut> {
+    constructor(ai: AxAIService, options?: Readonly<AxGenerateOptions>);
+    forward: (input: Readonly<AxRerankerIn>, options?: Readonly<AxProgramForwardOptions>) => Promise<AxRerankerOut>;
+}

package/docs/reranker.js

@@ -0,0 +1,26 @@
+import { AxGenerate, axStringUtil } from '../dsp/index.js';
+export class AxDefaultResultReranker extends AxGenerate {
+    constructor(ai, options) {
+        const signature = `"You are a re-ranker assistant tasked with evaluating a set of content items in relation to a specific question. Your role involves critically analyzing each content item to determine its relevance to the question and re-ranking them accordingly. This process includes assigning a relevance score from 0 to 10 to each content item based on how well it answers the question, its coverage of the topic, and the reliability of its information. This re-ranked list should start with the content item that is most relevant to the question and end with the least relevant. Output only the list."
+    query: string, items: string[] -> rankedItems: string[] "list of id, 5-words Rationale, relevance score"`;
+        super(ai, signature, options);
+    }
+    forward = async (input, options) => {
+        const { rankedItems } = await super.forward(input, options);
+        const sortedIndexes = rankedItems.map((item) => {
+            const { id: index } = axStringUtil.extractIdAndText(item);
+            return index;
+        });
+        // Ensure all elements are strings and filter out null or undefined
+        const sortedItems = input.items
+            .map((_, index) => {
+            const originalIndex = sortedIndexes[index];
+            return originalIndex !== undefined
+                ? input.items[originalIndex]
+                : undefined;
+        })
+            .filter((item) => item !== undefined);
+        return { rankedItems: sortedItems };
+    };
+}
+//# sourceMappingURL=reranker.js.map

package/docs/reranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../../docs/reranker.ts"],"names":[],"mappings":"AACA,OAAO,EACL,UAAU,EAGV,YAAY,EACb,MAAM,iBAAiB,CAAC;AAIzB,MAAM,OAAO,uBAAwB,SAAQ,UAG5C;IACC,YAAY,EAAe,EAAE,OAAqC;QAChE,MAAM,SAAS,GAAG;6GACuF,CAAC;QAE1G,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAEe,OAAO,GAAG,KAAK,EAC7B,KAA6B,EAC7B,OAA2C,EACnB,EAAE;QAC1B,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE5D,MAAM,aAAa,GAAa,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvD,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAC1D,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YAChB,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YAC3C,OAAO,aAAa,KAAK,SAAS;gBAChC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC;gBAC5B,CAAC,CAAC,SAAS,CAAC;QAChB,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QAExD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;IACtC,CAAC,CAAC;CACH"}

package/docs/rewriter.d.ts

@@ -0,0 +1,4 @@
+import { type AxAIService, AxGenerate, type AxGenerateOptions, type AxRewriteIn, type AxRewriteOut } from '../index.js';
+export declare class AxDefaultQueryRewriter extends AxGenerate<AxRewriteIn, AxRewriteOut> {
+    constructor(ai: AxAIService, options?: Readonly<AxGenerateOptions>);
+}

package/docs/rewriter.js

@@ -0,0 +1,9 @@
+import { AxGenerate } from '../index.js';
+export class AxDefaultQueryRewriter extends AxGenerate {
+    constructor(ai, options) {
+        const signature = `"You are a query rewriter assistant tasked with rewriting a given query to improve its clarity, specificity, and relevance. Your role involves analyzing the query to identify any ambiguities, generalizations, or irrelevant information and then rephrasing it to make it more focused and precise. The rewritten query should be concise, easy to understand, and directly related to the original query. Output only the rewritten query."
+    query: string -> rewrittenQuery: string`;
+        super(ai, signature, options);
+    }
+}
+//# sourceMappingURL=rewriter.js.map

package/docs/rewriter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rewriter.js","sourceRoot":"","sources":["../../docs/rewriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,UAAU,EAIX,MAAM,aAAa,CAAC;AAErB,MAAM,OAAO,sBAAuB,SAAQ,UAG3C;IACC,YAAY,EAAe,EAAE,OAAqC;QAChE,MAAM,SAAS,GAAG;4CACsB,CAAC;QAEzC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;CACF"}

package/docs/tika.d.ts

@@ -0,0 +1,18 @@
+export interface AxApacheTikaArgs {
+    url?: string | URL;
+    fetch?: typeof fetch;
+}
+export interface AxApacheTikaConvertOptions {
+    format?: 'text' | 'html';
+}
+export declare class AxApacheTika {
+    private tikaUrl;
+    private fetch?;
+    constructor(args?: Readonly<AxApacheTikaArgs>);
+    private _convert;
+    convert(files: Readonly<string[] | Blob[]>, options?: Readonly<{
+        batchSize?: number;
+        format?: 'html' | 'text';
+    }>): Promise<string[]>;
+}
+export default AxApacheTika;

package/docs/tika.js

@@ -0,0 +1,46 @@
+import { createReadStream } from 'node:fs';
+export class AxApacheTika {
+    tikaUrl;
+    fetch;
+    constructor(args) {
+        const _args = args ?? { url: 'http://localhost:9998/' };
+        this.tikaUrl = new URL('/tika', _args.url);
+        this.fetch = _args.fetch;
+    }
+    async _convert(file, options) {
+        const fileData = typeof file === 'string' ? createReadStream(file) : file.stream();
+        if (!fileData) {
+            throw new Error('Failed to read file data');
+        }
+        const acceptValue = options?.format === 'html' ? 'text/html' : 'text/plain';
+        try {
+            const res = await (this.fetch ?? fetch)(this.tikaUrl, {
+                body: fileData,
+                headers: { Accept: acceptValue },
+                duplex: 'half',
+                method: 'PUT'
+            });
+            if (!res.ok) {
+                throw new Error(`Failed to upload file: ${res.statusText}`);
+            }
+            const text = await res.text();
+            return text;
+        }
+        catch (error) {
+            throw new Error(`Error converting file: ${error}`);
+        }
+    }
+    async convert(files, options) {
+        const results = [];
+        const bs = options?.batchSize ?? 10;
+        for (let i = 0; i < files.length; i += bs) {
+            const batch = files.slice(i, i + bs);
+            const uploadPromises = batch.map((files) => this._convert(files, { format: options?.format }));
+            const batchResults = await Promise.all(uploadPromises);
+            results.push(...batchResults);
+        }
+        return results;
+    }
+}
+export default AxApacheTika;
+//# sourceMappingURL=tika.js.map

package/docs/tika.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tika.js","sourceRoot":"","sources":["../../docs/tika.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAW3C,MAAM,OAAO,YAAY;IACf,OAAO,CAAM;IACb,KAAK,CAAgB;IAE7B,YAAY,IAAiC;QAC3C,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,GAAG,EAAE,wBAAwB,EAAE,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,IAAmB,EACnB,OAA8C;QAE9C,MAAM,QAAQ,GACZ,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAEpE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC;QAE5E,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE;gBACpD,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE;gBAChC,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,KAAK;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;YAC9D,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,OAAO,CAClB,KAAkC,EAClC,OAAoE;QAEpE,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACzC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAClD,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACvD,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AACD,eAAe,YAAY,CAAC"}

package/dsp/asserts.d.ts

@@ -0,0 +1,32 @@
+import type { extractionState } from './extract.js';
+import type { AxSignature } from './sig.js';
+export interface AxAssertion {
+    fn(values: Record<string, unknown>): boolean | undefined;
+    message?: string;
+    optional?: boolean;
+}
+export interface AxStreamingAssertion {
+    fieldName: string;
+    fn(content: string): boolean | undefined;
+    message?: string;
+    optional?: boolean;
+}
+export declare class AxAssertionError extends Error {
+    private values;
+    private optional?;
+    constructor({ message, values, optional }: Readonly<{
+        message: string;
+        values: Record<string, unknown>;
+        optional?: boolean;
+    }>);
+    getValue: () => Record<string, unknown>;
+    getOptional: () => boolean | undefined;
+    getFixingInstructions: (_sig: Readonly<AxSignature>) => {
+        name: string;
+        title: string;
+        description: string;
+    }[];
+}
+export declare const assertAssertions: (asserts: readonly AxAssertion[], values: Record<string, unknown>) => void;
+export declare const assertStreamingAssertions: (asserts: readonly AxStreamingAssertion[], values: Record<string, unknown>, xstate: Readonly<extractionState>, content: string) => void;
+export declare const assertRequiredFields: (sig: Readonly<AxSignature>, values: Record<string, unknown>) => void;

package/dsp/asserts.js

@@ -0,0 +1,91 @@
+export class AxAssertionError extends Error {
+    values;
+    optional;
+    constructor({ message, values, optional }) {
+        super(message);
+        this.values = values;
+        this.optional = optional;
+        this.name = this.constructor.name;
+        this.stack = new Error().stack;
+    }
+    getValue = () => this.values;
+    getOptional = () => this.optional;
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    getFixingInstructions = (_sig) => {
+        const extraFields = [];
+        // for (const f of sig.getOutputFields()) {
+        //   extraFields.push({
+        //     name: `past_${f.name}`,
+        //     title: `Past ${f.title}`,
+        //     description: JSON.stringify(this.values[f.name])
+        //   });
+        // }
+        extraFields.push({
+            name: 'instructions',
+            title: 'Instructions',
+            description: this.message
+        });
+        return extraFields;
+    };
+}
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export const assertAssertions = (asserts, values) => {
+    for (const assert of asserts) {
+        const { fn, message, optional } = assert;
+        try {
+            const res = fn(values);
+            if (res === undefined) {
+                continue;
+            }
+            if (!res && message) {
+                throw new AxAssertionError({ message, values, optional });
+            }
+        }
+        catch (e) {
+            const message = e.message;
+            throw new AxAssertionError({ message, values, optional });
+        }
+    }
+};
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export const assertStreamingAssertions = (asserts, values, xstate, content) => {
+    if (!xstate.currField ||
+        xstate.s === -1 ||
+        !asserts ||
+        asserts.length === 0) {
+        return;
+    }
+    const fieldAsserts = asserts.filter((a) => a.fieldName === xstate.currField?.name);
+    if (fieldAsserts.length === 0) {
+        return;
+    }
+    const currValue = content.substring(xstate.s);
+    for (const assert of fieldAsserts) {
+        const { message, optional, fn } = assert;
+        try {
+            const res = fn(currValue);
+            if (res === undefined) {
+                continue;
+            }
+            if (!res && message) {
+                throw new AxAssertionError({ message, values, optional });
+            }
+        }
+        catch (e) {
+            const message = e.message;
+            throw new AxAssertionError({ message, values, optional });
+        }
+    }
+};
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export const assertRequiredFields = (sig, values) => {
+    const fields = sig.getOutputFields();
+    const missingFields = fields.filter((f) => !(f.name in values));
+    if (missingFields.length > 0) {
+        throw new AxAssertionError({
+            message: `Missing required fields: ${missingFields.map((f) => f.name).join(', ')}`,
+            values
+        });
+    }
+};
+//# sourceMappingURL=asserts.js.map

package/dsp/asserts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asserts.js","sourceRoot":"","sources":["../../dsp/asserts.ts"],"names":[],"mappings":"AAgBA,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACjC,MAAM,CAA0B;IAChC,QAAQ,CAAW;IAE3B,YAAY,EACV,OAAO,EACP,MAAM,EACN,QAAQ,EAKR;QACA,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC;IACjC,CAAC;IACM,QAAQ,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;IAC7B,WAAW,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;IAEzC,6DAA6D;IACtD,qBAAqB,GAAG,CAAC,IAA2B,EAAE,EAAE;QAC7D,MAAM,WAAW,GAAG,EAAE,CAAC;QAEvB,2CAA2C;QAC3C,uBAAuB;QACvB,8BAA8B;QAC9B,gCAAgC;QAChC,uDAAuD;QACvD,QAAQ;QACR,IAAI;QAEJ,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,cAAc;YACrB,WAAW,EAAE,IAAI,CAAC,OAAO;SAC1B,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC;CACH;AAED,gEAAgE;AAChE,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,OAA+B,EAC/B,MAA+B,EAC/B,EAAE;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,GAAG,IAAI,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,gBAAgB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAI,CAAW,CAAC,OAAO,CAAC;YACrC,MAAM,IAAI,gBAAgB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,gEAAgE;AAChE,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,OAAwC,EACxC,MAA+B,EAC/B,MAAiC,EACjC,OAAe,EACf,EAAE;IACF,IACE,CAAC,MAAM,CAAC,SAAS;QACjB,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;QACf,CAAC,OAAO;QACR,OAAO,CAAC,MAAM,KAAK,CAAC,EACpB,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAC9C,CAAC;IAEF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE9C,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC1B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,GAAG,IAAI,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,gBAAgB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAI,CAAW,CAAC,OAAO,CAAC;YACrC,MAAM,IAAI,gBAAgB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,gEAAgE;AAChE,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,GAA0B,EAC1B,MAA+B,EAC/B,EAAE;IACF,MAAM,MAAM,GAAG,GAAG,CAAC,eAAe,EAAE,CAAC;IACrC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC;IAChE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,gBAAgB,CAAC;YACzB,OAAO,EAAE,4BAA4B,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAClF,MAAM;SACP,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC"}

package/dsp/eval.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Calculates the Exact Match (EM) score between a prediction and ground truth.
+ *
+ * The EM score is a strict metric used in machine learning to assess if the predicted
+ * answer matches the ground truth exactly, commonly used in tasks like question answering.
+ *
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @returns A boolean indicating if the prediction exactly matches the ground truth.
+ */
+declare function emScore(prediction: string, groundTruth: string): boolean;
+/**
+ * Calculates the F1 score between a prediction and ground truth.
+ *
+ * The F1 score is a harmonic mean of precision and recall, widely used in NLP to measure
+ * a model's accuracy in considering both false positives and false negatives, offering a
+ * balance for evaluating classification models.
+ *
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @returns The F1 score as a number.
+ */
+declare function f1Score(prediction: string, groundTruth: string): number;
+/**
+ * Calculates a novel F1 score, taking into account a history of interaction and excluding stopwords.
+ *
+ * This metric extends the F1 score by considering contextual relevance and filtering out common words
+ * that might skew the assessment of the prediction's quality, especially in conversational models or
+ * when historical context is relevant.
+ *
+ * @param history The historical context or preceding interactions.
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @param returnRecall Optionally return the recall score instead of F1.
+ * @returns The novel F1 or recall score as a number.
+ */
+declare function novelF1ScoreOptimized(history: string, prediction: string, groundTruth: string, returnRecall?: boolean): number;
+export declare const axEvalUtil: {
+    emScore: typeof emScore;
+    f1Score: typeof f1Score;
+    novelF1ScoreOptimized: typeof novelF1ScoreOptimized;
+};
+export {};

package/dsp/eval.js

@@ -0,0 +1,125 @@
+import { stopwords } from './stopwords.js';
+/**
+ * Filters out tokens based on a set of exclusion tokens.
+ *
+ * @param tokens The array of tokens to filter.
+ * @param exclusions A set containing tokens to exclude.
+ * @returns An array of filtered tokens.
+ */
+function filterTokens(tokens, exclusions) {
+    return tokens.filter((token) => !exclusions.has(token));
+}
+/**
+ * Counts the occurrences of each token in an array of tokens.
+ *
+ * This function supports the preprocessing step for NLP tasks like text similarity
+ * and classification by transforming text into a bag-of-words model, facilitating
+ * the comparison of different texts based on their content.
+ *
+ * @param tokens An array of string tokens.
+ * @returns A Counter object mapping each token to its count.
+ */
+function countTokens(tokens) {
+    const counter = {};
+    for (const token of tokens) {
+        counter[token] = (counter[token] || 0) + 1;
+    }
+    return counter;
+}
+/**
+ * Normalizes text by lowercasing, removing punctuation, and squashing multiple spaces.
+ *
+ * This normalization is crucial in NLP for reducing the complexity of the text data,
+ * minimizing the variance between words that should be considered the same for analysis
+ * purposes (e.g., "Dog!" and "dog" are treated as the same word).
+ *
+ * @param s A string to be normalized.
+ * @returns A normalized string.
+ */
+function normalizeText(s) {
+    s = s.normalize('NFD');
+    s = s.replace(/\b(a|an|the)\b/g, ' ');
+    s = s.split(/\s+/).join(' ');
+    s = s.replace(/[!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~]/g, '');
+    return s.toLowerCase();
+}
+/**
+ * Calculates the Exact Match (EM) score between a prediction and ground truth.
+ *
+ * The EM score is a strict metric used in machine learning to assess if the predicted
+ * answer matches the ground truth exactly, commonly used in tasks like question answering.
+ *
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @returns A boolean indicating if the prediction exactly matches the ground truth.
+ */
+function emScore(prediction, groundTruth) {
+    return normalizeText(prediction) === normalizeText(groundTruth);
+}
+/**
+ * Calculates the F1 score between a prediction and ground truth.
+ *
+ * The F1 score is a harmonic mean of precision and recall, widely used in NLP to measure
+ * a model's accuracy in considering both false positives and false negatives, offering a
+ * balance for evaluating classification models.
+ *
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @returns The F1 score as a number.
+ */
+function f1Score(prediction, groundTruth) {
+    const predictionTokens = normalizeText(prediction).split(' ');
+    const groundTruthTokens = normalizeText(groundTruth).split(' ');
+    // Calculate the intersection of common tokens between prediction and ground truth
+    const predictionCounts = countTokens(predictionTokens);
+    const groundTruthCounts = countTokens(groundTruthTokens);
+    let numSame = 0;
+    for (const token in predictionCounts) {
+        const v1 = predictionCounts[token] ?? 0;
+        const v2 = groundTruthCounts[token] ?? 0;
+        numSame += Math.min(v1, v2);
+    }
+    if (numSame === 0) {
+        return 0;
+    }
+    const precision = numSame / predictionTokens.length;
+    const recall = numSame / groundTruthTokens.length;
+    return (2 * precision * recall) / (precision + recall);
+}
+/**
+ * Calculates a novel F1 score, taking into account a history of interaction and excluding stopwords.
+ *
+ * This metric extends the F1 score by considering contextual relevance and filtering out common words
+ * that might skew the assessment of the prediction's quality, especially in conversational models or
+ * when historical context is relevant.
+ *
+ * @param history The historical context or preceding interactions.
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @param returnRecall Optionally return the recall score instead of F1.
+ * @returns The novel F1 or recall score as a number.
+ */
+function novelF1ScoreOptimized(history, prediction, groundTruth, returnRecall = false) {
+    // Normalize and split the input texts into tokens
+    const historyTokens = normalizeText(history).split(' ');
+    let predictionTokens = normalizeText(prediction).split(' ');
+    let groundTruthTokens = normalizeText(groundTruth).split(' ');
+    // Combine stopwords and history tokens for exclusion
+    const exclusions = new Set([...stopwords, ...historyTokens]);
+    // Filter prediction and ground truth tokens against the exclusions
+    predictionTokens = filterTokens(predictionTokens, exclusions);
+    groundTruthTokens = filterTokens(groundTruthTokens, exclusions);
+    // Proceed with calculating common tokens, precision, recall, and F1 score as previously outlined
+    // Placeholder for the calculation logic
+    const numSame = 0; // This should be calculated as before
+    const precision = numSame / predictionTokens.length;
+    const recall = numSame / groundTruthTokens.length;
+    const f1 = (2 * precision * recall) / (precision + recall);
+    return returnRecall ? recall : f1;
+}
+export const axEvalUtil = {
+    emScore,
+    f1Score,
+    novelF1ScoreOptimized
+};
+//# sourceMappingURL=eval.js.map