@ax-llm/ax 9.0.0

package/build/module/src/docs/manager.d.ts

@@ -0,0 +1,45 @@
+import { type AIService, type DBService, type Program } from '../index.js';
+export type RewriteIn = {
+    query: string;
+};
+export type RewriteOut = {
+    rewrittenQuery: string;
+};
+export type RerankerIn = {
+    query: string;
+    items: string[];
+};
+export type RerankerOut = {
+    rankedItems: string[];
+};
+export interface DBLoaderOptions {
+    chunker?: (text: string) => string[];
+    rewriter?: Program<RewriteIn, RewriteOut>;
+    reranker?: Program<RerankerIn, RerankerOut>;
+}
+export interface DBManagerArgs {
+    ai: AIService;
+    db: DBService;
+    config?: DBLoaderOptions;
+}
+export interface Match {
+    score: number;
+    text: string;
+}
+export declare class DBManager {
+    private ai;
+    private db;
+    private chunker;
+    private rewriter?;
+    private reranker?;
+    constructor({ ai, db, config }: Readonly<DBManagerArgs>);
+    private defaultChunker;
+    insert: (text: Readonly<string | string[]>, options?: Readonly<{
+        batchSize?: number;
+        maxWordsPerChunk?: number;
+        minWordsPerChunk?: number;
+    }>) => Promise<void>;
+    query: (query: Readonly<string | string[] | number | number[]>, { topPercent }?: Readonly<{
+        topPercent?: number;
+    }> | undefined) => Promise<Match[][]>;
+}

package/build/module/src/docs/manager.js

@@ -0,0 +1,159 @@
+import {} from '../index.js';
+const table = '_internal';
+export class DBManager {
+    ai;
+    db;
+    chunker;
+    rewriter;
+    reranker;
+    constructor({ ai, db, config }) {
+        this.ai = ai;
+        this.db = db;
+        this.chunker = config?.chunker ?? this.defaultChunker;
+        this.reranker = config?.reranker;
+        this.rewriter = config?.rewriter;
+    }
+    defaultChunker = (text) => {
+        // Default chunking by paragraphs
+        return text.split(/\n\n+/);
+    };
+    insert = async (text, options) => {
+        try {
+            const chunkerInput = Array.isArray(text)
+                ? text.join('\n\n')
+                : text;
+            // Chunk the text using the specified or default chunking function
+            const initialChunks = this.chunker(chunkerInput).filter((chunk) => chunk.length > 0);
+            const maxWordsPerChunk = options?.maxWordsPerChunk;
+            const minWordsPerChunk = options?.minWordsPerChunk;
+            const chunks = processChunks({
+                initialChunks,
+                minWordsPerChunk,
+                maxWordsPerChunk
+            });
+            const bs = options?.batchSize ?? 10;
+            // Process chunks in batches of 10
+            for (let i = 0; i < chunks.length; i += bs) {
+                const batch = chunks.slice(i, i + bs);
+                // Get embeddings for the whole batch from the AI service in one call
+                const ret = await this.ai.embed({ texts: batch });
+                // Prepare batch for bulk upsert
+                const embeddings = ret.embeddings
+                    .map((embedding, index) => ({
+                    id: `chunk_${Date.now() + index}`, // Unique ID for each chunk, adjusted by index
+                    table,
+                    values: embedding,
+                    metadata: { text: batch[index] ?? '' }
+                }))
+                    .filter((v) => v.metadata?.text && v.metadata?.text.length > 0);
+                // Batch upsert embeddings
+                await this.db.batchUpsert(embeddings);
+            }
+        }
+        catch (error) {
+            throw new Error(`Error processing text: ${error}`);
+        }
+    };
+    query = async (query, { topPercent } = {}) => {
+        const texts = Array.isArray(query) ? query : [query];
+        if (typeof texts[0] === 'string' && this.rewriter) {
+            for (const [i, text] of texts.entries()) {
+                const { rewrittenQuery } = await this.rewriter.forward({
+                    query: text
+                });
+                texts[i] = rewrittenQuery;
+            }
+        }
+        let queries;
+        if (typeof texts[0] === 'string') {
+            const embedResults = await this.ai.embed({ texts });
+            queries = embedResults.embeddings.map((values) => this.db.query({ table, values }));
+        }
+        else {
+            queries = texts.map((values) => this.db.query({ table, values }));
+        }
+        const queryResults = await Promise.all(queries);
+        const res = [];
+        for (const { matches } of queryResults) {
+            const m = matches
+                .filter((v) => v.metadata?.text && v.metadata?.text.length > 0)
+                .map(({ score, metadata }) => ({ score, text: metadata?.text ?? '' }));
+            const tp = topPercent && topPercent > 1 ? topPercent / 100 : topPercent;
+            const resultItems = tp ? getTopInPercent(m, tp) : m;
+            if (this.reranker) {
+                const { rankedItems } = await this.reranker.forward({
+                    query: texts[0],
+                    items: resultItems.map((item) => item.text)
+                });
+                const items = rankedItems
+                    .map((item) => resultItems.find((r) => r.text === item))
+                    .filter((v) => v !== undefined);
+                res.push(items);
+            }
+            else {
+                res.push(resultItems);
+            }
+        }
+        return res;
+    };
+}
+const processChunks = ({ initialChunks, maxWordsPerChunk = 350, minWordsPerChunk = 250 }) => {
+    const chunks = [];
+    let currentChunk = '';
+    let currentWordCount = 0;
+    initialChunks.forEach((chunk) => {
+        const words = chunk.split(/\s+/); // Split the chunk into words
+        const wordCount = words.length; // Count words in the current chunk
+        if (currentWordCount + wordCount <= maxWordsPerChunk) {
+            // Add to the current chunk if within the max size limit
+            currentChunk += chunk + '\n\n';
+            currentWordCount += wordCount;
+        }
+        else if (currentWordCount > 0 &&
+            currentWordCount + wordCount <= maxWordsPerChunk * 1.5) {
+            // If the total word count exceeds the limit but is less than 150% of the maxWordsPerChunk
+            currentChunk += chunk + '\n\n';
+            currentWordCount += wordCount;
+        }
+        else {
+            // If the current chunk is not empty and adding the new chunk exceeds the adjusted limit
+            if (currentWordCount > minWordsPerChunk) {
+                chunks.push(currentChunk.trim());
+                currentChunk = '';
+                currentWordCount = 0;
+            }
+            // Handle the case where the chunk itself is larger than the limit
+            if (wordCount > maxWordsPerChunk) {
+                const remainingWords = words;
+                while (remainingWords.length > maxWordsPerChunk * 1.5) {
+                    const slice = remainingWords.splice(0, maxWordsPerChunk);
+                    chunks.push(slice.join(' '));
+                }
+                // Add the last portion if it fits the condition of being within 150% of maxWordsPerChunk
+                if (remainingWords.length > 0) {
+                    currentChunk += remainingWords.join(' ') + '\n\n';
+                    currentWordCount += remainingWords.length;
+                }
+            }
+            else {
+                // If the new chunk is smaller than the maximum words per chunk
+                currentChunk = chunk + '\n\n';
+                currentWordCount = wordCount;
+            }
+        }
+    });
+    // Push the last chunk if it exists and meets the minimum words condition
+    if (currentWordCount > minWordsPerChunk || chunks.length === 0) {
+        chunks.push(currentChunk.trim());
+    }
+    return chunks;
+};
+const getTopInPercent = (entries, percent = 0.1) => {
+    // Sort entries by score in ascending order
+    const sortedEntries = [...entries].sort((a, b) => a.score - b.score);
+    // Calculate the number of entries to take (top 10%)
+    const topTenPercentCount = Math.ceil(sortedEntries.length * percent);
+    // Return the top 10% of entries
+    return sortedEntries.slice(0, topTenPercentCount);
+};
+//# sourceMappingURL=manager.js.map

package/build/module/src/docs/manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../../../src/docs/manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAKN,MAAM,aAAa,CAAC;AAyBrB,MAAM,KAAK,GAAG,WAAW,CAAC;AAE1B,MAAM,OAAO,SAAS;IACZ,EAAE,CAAY;IACd,EAAE,CAAY;IACd,OAAO,CAA6B;IACpC,QAAQ,CAAkC;IAC1C,QAAQ,CAAoC;IAEpD,YAAY,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,EAA2B;QACrD,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,QAAQ,GAAG,MAAM,EAAE,QAAQ,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,MAAM,EAAE,QAAQ,CAAC;IACnC,CAAC;IAEO,cAAc,GAAG,CAAC,IAAY,EAAY,EAAE;QAClD,iCAAiC;QACjC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,GAAG,KAAK,EACZ,IAAiC,EACjC,OAIE,EACa,EAAE;QACjB,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBACtC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;gBACnB,CAAC,CAAE,IAAe,CAAC;YAErB,kEAAkE;YAClE,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,MAAM,CACrD,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAC5B,CAAC;YAEF,MAAM,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,CAAC;YACnD,MAAM,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,CAAC;YAEnD,MAAM,MAAM,GAAG,aAAa,CAAC;gBAC3B,aAAa;gBACb,gBAAgB;gBAChB,gBAAgB;aACjB,CAAC,CAAC;YAEH,MAAM,EAAE,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;YAEpC,kCAAkC;YAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;gBAEtC,qEAAqE;gBACrE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;gBAElD,gCAAgC;gBAChC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU;qBAC9B,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1B,EAAE,EAAE,SAAS,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,EAAE,8CAA8C;oBACjF,KAAK;oBACL,MAAM,EAAE,SAAS;oBACjB,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE;iBACvC,CAAC,CAAC;qBACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAElE,0BAA0B;gBAC1B,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,GAAG,KAAK,EACX,KAAsD,EACtD,EAAE,UAAU,KAAoD,EAAE,EAC9C,EAAE;QACtB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAErD,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClD,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBACxC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;oBACrD,KAAK,EAAE,IAAI;iBACZ,CAAC,CAAC;gBACH,KAAK,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,OAAmC,CAAC;QAExC,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;YACpD,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAC/C,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CACjC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,GAAG,GAAc,EAAE,CAAC;QAE1B,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,YAAY,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,OAAO;iBACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,IAAI,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;iBAC9D,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;YAEzE,MAAM,EAAE,GAAG,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;YACxE,MAAM,WAAW,GAAG,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEpD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAClD,KAAK,EAAE,KAAK,CAAC,CAAC,CAAW;oBACzB,KAAK,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;iBAC5C,CAAC,CAAC;gBAEH,MAAM,KAAK,GAAG,WAAW;qBACtB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;qBACvD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAY,CAAC;gBAE7C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;CACH;AAED,MAAM,aAAa,GAAG,CAAC,EACrB,aAAa,EACb,gBAAgB,GAAG,GAAG,EACtB,gBAAgB,GAAG,GAAG,EAKtB,EAAY,EAAE;IACd,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,aAAa,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,6BAA6B;QAC/D,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,mCAAmC;QAEnE,IAAI,gBAAgB,GAAG,SAAS,IAAI,gBAAgB,EAAE,CAAC;YACrD,wDAAwD;YACxD,YAAY,IAAI,KAAK,GAAG,MAAM,CAAC;YAC/B,gBAAgB,IAAI,SAAS,CAAC;QAChC,CAAC;aAAM,IACL,gBAAgB,GAAG,CAAC;YACpB,gBAAgB,GAAG,SAAS,IAAI,gBAAgB,GAAG,GAAG,EACtD,CAAC;YACD,0FAA0F;YAC1F,YAAY,IAAI,KAAK,GAAG,MAAM,CAAC;YAC/B,gBAAgB,IAAI,SAAS,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,wFAAwF;YACxF,IAAI,gBAAgB,GAAG,gBAAgB,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjC,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,CAAC;YACvB,CAAC;YACD,kEAAkE;YAClE,IAAI,SAAS,GAAG,gBAAgB,EAAE,CAAC;gBACjC,MAAM,cAAc,GAAG,KAAK,CAAC;gBAC7B,OAAO,cAAc,CAAC,MAAM,GAAG,gBAAgB,GAAG,GAAG,EAAE,CAAC;oBACtD,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;oBACzD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/B,CAAC;gBACD,yFAAyF;gBACzF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,YAAY,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;oBAClD,gBAAgB,IAAI,cAAc,CAAC,MAAM,CAAC;gBAC5C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,+DAA+D;gBAC/D,YAAY,GAAG,KAAK,GAAG,MAAM,CAAC;gBAC9B,gBAAgB,GAAG,SAAS,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,IAAI,gBAAgB,GAAG,gBAAgB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/D,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CACtB,OAAyB,EACzB,UAAkB,GAAG,EACZ,EAAE;IACX,2CAA2C;IAC3C,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAErE,oDAAoD;IACpD,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC;IAErE,gCAAgC;IAChC,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;AACpD,CAAC,CAAC"}

package/build/module/src/docs/reranker.d.ts

@@ -0,0 +1,7 @@
+import { Generate, type GenerateOptions, type ProgramForwardOptions } from '../dsp/index.js';
+import type { AIService } from '../text/index.js';
+import type { RerankerIn, RerankerOut } from './manager.js';
+export declare class DefaultResultReranker extends Generate<RerankerIn, RerankerOut> {
+    constructor(ai: AIService, options?: Readonly<GenerateOptions>);
+    forward: (input: Readonly<RerankerIn>, options?: Readonly<ProgramForwardOptions>) => Promise<RerankerOut>;
+}

package/build/module/src/docs/reranker.js

@@ -0,0 +1,26 @@
+import { extractIdAndText, Generate } from '../dsp/index.js';
+export class DefaultResultReranker extends Generate {
+    constructor(ai, options) {
+        const signature = `"You are a re-ranker assistant tasked with evaluating a set of content items in relation to a specific question. Your role involves critically analyzing each content item to determine its relevance to the question and re-ranking them accordingly. This process includes assigning a relevance score from 0 to 10 to each content item based on how well it answers the question, its coverage of the topic, and the reliability of its information. This re-ranked list should start with the content item that is most relevant to the question and end with the least relevant. Output only the list."
+    query: string, items: string[] -> rankedItems: string[] "list of id, 5-words Rationale, relevance score"`;
+        super(ai, signature, options);
+    }
+    forward = async (input, options) => {
+        const { rankedItems } = await super.forward(input, options);
+        const sortedIndexes = rankedItems.map((item) => {
+            const { id: index } = extractIdAndText(item);
+            return index;
+        });
+        // Ensure all elements are strings and filter out null or undefined
+        const sortedItems = input.items
+            .map((_, index) => {
+            const originalIndex = sortedIndexes[index];
+            return originalIndex !== undefined
+                ? input.items[originalIndex]
+                : undefined;
+        })
+            .filter((item) => item !== undefined);
+        return { rankedItems: sortedItems };
+    };
+}
+//# sourceMappingURL=reranker.js.map

package/build/module/src/docs/reranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../../../../src/docs/reranker.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,QAAQ,EAGT,MAAM,iBAAiB,CAAC;AAKzB,MAAM,OAAO,qBAAsB,SAAQ,QAAiC;IAC1E,YAAY,EAAa,EAAE,OAAmC;QAC5D,MAAM,SAAS,GAAG;6GACuF,CAAC;QAE1G,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAEe,OAAO,GAAG,KAAK,EAC7B,KAA2B,EAC3B,OAAyC,EACnB,EAAE;QACxB,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE5D,MAAM,aAAa,GAAa,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvD,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAC7C,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YAChB,MAAM,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YAC3C,OAAO,aAAa,KAAK,SAAS;gBAChC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC;gBAC5B,CAAC,CAAC,SAAS,CAAC;QAChB,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QAExD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;IACtC,CAAC,CAAC;CACH"}

package/build/module/src/docs/rewriter.d.ts

@@ -0,0 +1,4 @@
+import { type AIService, Generate, type GenerateOptions, type RewriteIn, type RewriteOut } from '../index.js';
+export declare class DefaultQueryRewriter extends Generate<RewriteIn, RewriteOut> {
+    constructor(ai: AIService, options?: Readonly<GenerateOptions>);
+}

package/build/module/src/docs/rewriter.js

@@ -0,0 +1,9 @@
+import { Generate } from '../index.js';
+export class DefaultQueryRewriter extends Generate {
+    constructor(ai, options) {
+        const signature = `"You are a query rewriter assistant tasked with rewriting a given query to improve its clarity, specificity, and relevance. Your role involves analyzing the query to identify any ambiguities, generalizations, or irrelevant information and then rephrasing it to make it more focused and precise. The rewritten query should be concise, easy to understand, and directly related to the original query. Output only the rewritten query."
+    query: string -> rewrittenQuery: string`;
+        super(ai, signature, options);
+    }
+}
+//# sourceMappingURL=rewriter.js.map

package/build/module/src/docs/rewriter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rewriter.js","sourceRoot":"","sources":["../../../../src/docs/rewriter.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAIT,MAAM,aAAa,CAAC;AAErB,MAAM,OAAO,oBAAqB,SAAQ,QAA+B;IACvE,YAAY,EAAa,EAAE,OAAmC;QAC5D,MAAM,SAAS,GAAG;4CACsB,CAAC;QAEzC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;CACF"}

package/build/module/src/docs/tika.d.ts

@@ -0,0 +1,21 @@
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+export interface ApacheTikaArgs {
+    url?: string | URL;
+    fetch?: typeof fetch;
+}
+export interface ApacheTikaConvertOptions {
+    format?: 'text' | 'html';
+}
+export declare class ApacheTika {
+    private tikaUrl;
+    private fetch?;
+    constructor(args?: Readonly<ApacheTikaArgs>);
+    private _convert;
+    convert(files: Readonly<string[] | Blob[]>, options?: Readonly<{
+        batchSize?: number;
+        format?: 'html' | 'text';
+    }>): Promise<string[]>;
+}
+export default ApacheTika;

package/build/module/src/docs/tika.js

@@ -0,0 +1,46 @@
+import { createReadStream } from 'fs';
+export class ApacheTika {
+    tikaUrl;
+    fetch;
+    constructor(args) {
+        const _args = args ?? { url: 'http://localhost:9998/' };
+        this.tikaUrl = new URL('/tika', _args.url);
+        this.fetch = _args.fetch;
+    }
+    async _convert(file, options) {
+        const fileData = typeof file === 'string' ? createReadStream(file) : file.stream();
+        if (!fileData) {
+            throw new Error('Failed to read file data');
+        }
+        const acceptValue = options?.format === 'html' ? 'text/html' : 'text/plain';
+        try {
+            const res = await (this.fetch ?? fetch)(this.tikaUrl, {
+                body: fileData,
+                headers: { Accept: acceptValue },
+                duplex: 'half',
+                method: 'PUT'
+            });
+            if (!res.ok) {
+                throw new Error(`Failed to upload file: ${res.statusText}`);
+            }
+            const text = await res.text();
+            return text;
+        }
+        catch (error) {
+            throw new Error(`Error converting file: ${error}`);
+        }
+    }
+    async convert(files, options) {
+        const results = [];
+        const bs = options?.batchSize ?? 10;
+        for (let i = 0; i < files.length; i += bs) {
+            const batch = files.slice(i, i + bs);
+            const uploadPromises = batch.map((files) => this._convert(files, { format: options?.format }));
+            const batchResults = await Promise.all(uploadPromises);
+            results.push(...batchResults);
+        }
+        return results;
+    }
+}
+export default ApacheTika;
+//# sourceMappingURL=tika.js.map

package/build/module/src/docs/tika.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tika.js","sourceRoot":"","sources":["../../../../src/docs/tika.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AAWtC,MAAM,OAAO,UAAU;IACb,OAAO,CAAM;IACb,KAAK,CAAgB;IAE7B,YAAY,IAA+B;QACzC,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,GAAG,EAAE,wBAAwB,EAAE,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,IAAmB,EACnB,OAA4C;QAE5C,MAAM,QAAQ,GACZ,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAEpE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,WAAW,GAAG,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC;QAE5E,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE;gBACpD,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE;gBAChC,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,KAAK;aACd,CAAC,CAAC;YAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;YAC9D,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,OAAO,CAClB,KAAkC,EAClC,OAAoE;QAEpE,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACzC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAClD,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACvD,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,eAAe,UAAU,CAAC"}

package/build/module/src/dsp/asserts.d.ts

@@ -0,0 +1,32 @@
+import type { extractionState } from './extract.js';
+import type { Signature } from './sig.js';
+export interface Assertion {
+    fn(values: Record<string, unknown>): boolean | undefined;
+    message?: string;
+    optional?: boolean;
+}
+export interface StreamingAssertion {
+    fieldName: string;
+    fn(content: string): boolean | undefined;
+    message?: string;
+    optional?: boolean;
+}
+export declare class AssertionError extends Error {
+    private values;
+    private optional?;
+    constructor({ message, values, optional }: Readonly<{
+        message: string;
+        values: Record<string, unknown>;
+        optional?: boolean;
+    }>);
+    getValue: () => Record<string, unknown>;
+    getOptional: () => boolean | undefined;
+    getFixingInstructions: (_sig: Readonly<Signature>) => {
+        name: string;
+        title: string;
+        description: string;
+    }[];
+}
+export declare const assertAssertions: (asserts: readonly Assertion[], values: Record<string, unknown>) => void;
+export declare const assertStreamingAssertions: (asserts: readonly StreamingAssertion[], values: Record<string, unknown>, xstate: Readonly<extractionState>, content: string) => void;
+export declare const assertRequiredFields: (sig: Readonly<Signature>, values: Record<string, unknown>) => void;

package/build/module/src/dsp/asserts.js

@@ -0,0 +1,88 @@
+export class AssertionError extends Error {
+    values;
+    optional;
+    constructor({ message, values, optional }) {
+        super(message);
+        this.values = values;
+        this.optional = optional;
+        this.name = this.constructor.name;
+        this.stack = new Error().stack;
+    }
+    getValue = () => this.values;
+    getOptional = () => this.optional;
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    getFixingInstructions = (_sig) => {
+        const extraFields = [];
+        // for (const f of sig.getOutputFields()) {
+        //   extraFields.push({
+        //     name: `past_${f.name}`,
+        //     title: `Past ${f.title}`,
+        //     description: JSON.stringify(this.values[f.name])
+        //   });
+        // }
+        extraFields.push({
+            name: 'instructions',
+            title: 'Instructions',
+            description: this.message
+        });
+        return extraFields;
+    };
+}
+export const assertAssertions = (asserts, values) => {
+    for (const assert of asserts) {
+        const { fn, message, optional } = assert;
+        try {
+            const res = fn(values);
+            if (res === undefined) {
+                continue;
+            }
+            if (!res && message) {
+                throw new AssertionError({ message, values, optional });
+            }
+        }
+        catch (e) {
+            const message = e.message;
+            throw new AssertionError({ message, values, optional });
+        }
+    }
+};
+export const assertStreamingAssertions = (asserts, values, xstate, content) => {
+    if (!xstate.currField ||
+        xstate.s === -1 ||
+        !asserts ||
+        asserts.length === 0) {
+        return;
+    }
+    const fieldAsserts = asserts.filter((a) => a.fieldName === xstate.currField?.name);
+    if (fieldAsserts.length === 0) {
+        return;
+    }
+    const currValue = content.substring(xstate.s);
+    for (const assert of fieldAsserts) {
+        const { message, optional, fn } = assert;
+        try {
+            const res = fn(currValue);
+            if (res === undefined) {
+                continue;
+            }
+            if (!res && message) {
+                throw new AssertionError({ message, values, optional });
+            }
+        }
+        catch (e) {
+            const message = e.message;
+            throw new AssertionError({ message, values, optional });
+        }
+    }
+};
+export const assertRequiredFields = (sig, values) => {
+    const fields = sig.getOutputFields();
+    const missingFields = fields.filter((f) => !(f.name in values));
+    if (missingFields.length > 0) {
+        throw new AssertionError({
+            message: `Missing required fields: ${missingFields.map((f) => f.name).join(', ')}`,
+            values
+        });
+    }
+};
+//# sourceMappingURL=asserts.js.map

package/build/module/src/dsp/asserts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asserts.js","sourceRoot":"","sources":["../../../../src/dsp/asserts.ts"],"names":[],"mappings":"AAgBA,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC/B,MAAM,CAA0B;IAChC,QAAQ,CAAW;IAE3B,YAAY,EACV,OAAO,EACP,MAAM,EACN,QAAQ,EAKR;QACA,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC;IACjC,CAAC;IACM,QAAQ,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;IAC7B,WAAW,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;IAEzC,6DAA6D;IACtD,qBAAqB,GAAG,CAAC,IAAyB,EAAE,EAAE;QAC3D,MAAM,WAAW,GAAG,EAAE,CAAC;QAEvB,2CAA2C;QAC3C,uBAAuB;QACvB,8BAA8B;QAC9B,gCAAgC;QAChC,uDAAuD;QACvD,QAAQ;QACR,IAAI;QAEJ,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,cAAc;YACrB,WAAW,EAAE,IAAI,CAAC,OAAO;SAC1B,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC;IACrB,CAAC,CAAC;CACH;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,OAA6B,EAC7B,MAA+B,EAC/B,EAAE;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;YACvB,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,GAAG,IAAI,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,cAAc,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAI,CAAW,CAAC,OAAO,CAAC;YACrC,MAAM,IAAI,cAAc,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAAG,CACvC,OAAsC,EACtC,MAA+B,EAC/B,MAAiC,EACjC,OAAe,EACf,EAAE;IACF,IACE,CAAC,MAAM,CAAC,SAAS;QACjB,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;QACf,CAAC,OAAO;QACR,OAAO,CAAC,MAAM,KAAK,CAAC,EACpB,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAC9C,CAAC;IAEF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE9C,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC1B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACtB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,GAAG,IAAI,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,cAAc,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAI,CAAW,CAAC,OAAO,CAAC;YACrC,MAAM,IAAI,cAAc,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,GAAwB,EACxB,MAA+B,EAC/B,EAAE;IACF,MAAM,MAAM,GAAG,GAAG,CAAC,eAAe,EAAE,CAAC;IACrC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC;IAChE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,cAAc,CAAC;YACvB,OAAO,EAAE,4BAA4B,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAClF,MAAM;SACP,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC"}

package/build/module/src/dsp/eval.d.ts

@@ -0,0 +1,62 @@
+export type Counter = {
+    [key: string]: number;
+};
+/**
+ * Counts the occurrences of each token in an array of tokens.
+ *
+ * This function supports the preprocessing step for NLP tasks like text similarity
+ * and classification by transforming text into a bag-of-words model, facilitating
+ * the comparison of different texts based on their content.
+ *
+ * @param tokens An array of string tokens.
+ * @returns A Counter object mapping each token to its count.
+ */
+export declare function countTokens(tokens: readonly string[]): Counter;
+/**
+ * Normalizes text by lowercasing, removing punctuation, and squashing multiple spaces.
+ *
+ * This normalization is crucial in NLP for reducing the complexity of the text data,
+ * minimizing the variance between words that should be considered the same for analysis
+ * purposes (e.g., "Dog!" and "dog" are treated as the same word).
+ *
+ * @param s A string to be normalized.
+ * @returns A normalized string.
+ */
+export declare function normalizeText(s: string): string;
+/**
+ * Calculates the Exact Match (EM) score between a prediction and ground truth.
+ *
+ * The EM score is a strict metric used in machine learning to assess if the predicted
+ * answer matches the ground truth exactly, commonly used in tasks like question answering.
+ *
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @returns A boolean indicating if the prediction exactly matches the ground truth.
+ */
+export declare function emScore(prediction: string, groundTruth: string): boolean;
+/**
+ * Calculates the F1 score between a prediction and ground truth.
+ *
+ * The F1 score is a harmonic mean of precision and recall, widely used in NLP to measure
+ * a model's accuracy in considering both false positives and false negatives, offering a
+ * balance for evaluating classification models.
+ *
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @returns The F1 score as a number.
+ */
+export declare function f1Score(prediction: string, groundTruth: string): number;
+/**
+ * Calculates a novel F1 score, taking into account a history of interaction and excluding stopwords.
+ *
+ * This metric extends the F1 score by considering contextual relevance and filtering out common words
+ * that might skew the assessment of the prediction's quality, especially in conversational models or
+ * when historical context is relevant.
+ *
+ * @param history The historical context or preceding interactions.
+ * @param prediction The predicted text.
+ * @param groundTruth The actual correct text.
+ * @param returnRecall Optionally return the recall score instead of F1.
+ * @returns The novel F1 or recall score as a number.
+ */
+export declare function novelF1ScoreOptimized(history: string, prediction: string, groundTruth: string, returnRecall?: boolean): number;