@tyvm/knowhow 0.0.1

package/ts_build/src/dataset/diffs/test.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+require("source-map-support/register");
+const diff_1 = require("diff");
+const patcher_1 = require("../../agents/patcher/patcher");
+const utils_1 = require("../../utils");
+const patch_1 = require("../../agents/tools/patch");
+const hashes_1 = require("../../hashes");
+const dataset = [];
+class PatchTestAgent extends patcher_1.PatchingAgent {
+    name = "PatchTestAgent";
+    async getInitialMessages(userInput) {
+        const baseMessages = await super.getInitialMessages(userInput);
+        baseMessages.push({
+            content: "We are testing the patch tool, attempt to use only that tool with the input provided in the user's message.",
+            role: "user",
+        });
+        return baseMessages;
+    }
+    async processToolMessages(toolCall) {
+        const toolMessages = await super.processToolMessages(toolCall);
+        toolMessages.push({
+            name: "finalAnswer",
+            content: "Done",
+        });
+        return toolMessages;
+    }
+}
+async function testDataset() {
+    const patchAgent = new PatchTestAgent();
+    let successCount = 0;
+    let attempts = 0;
+    const testHash = "";
+    const shuffled = dataset
+        .map((value) => ({ value, sort: Math.random() }))
+        .sort((a, b) => a.sort - b.sort)
+        .map(({ value }) => value);
+    for (const patchData of shuffled) {
+        const hash = await (0, hashes_1.md5Hash)(patchData.patch);
+        if (testHash && testHash !== hash) {
+            continue;
+        }
+        const changes = (0, patch_1.parseHunks)(patchData.patch);
+        const totalChanges = changes
+            .map((hunk) => hunk.additions.length + hunk.subtractions.length)
+            .reduce((a, b) => a + b, 0);
+        if (totalChanges > 10) {
+            continue;
+        }
+        async function testPatchFile(filePath, patch) {
+            attempts++;
+            const originalContent = patchData.before;
+            const fixedPatch = (0, patch_1.fixPatch)(patchData.before, patch);
+            const { validHunks, invalidHunks } = (0, patch_1.categorizeHunks)(originalContent, patch);
+            const validPatch = (0, patch_1.hunksToPatch)(validHunks);
+            const updatedContent = (0, diff_1.applyPatch)(originalContent, validPatch);
+            const success = !!updatedContent && validHunks.length;
+            await saveToolDiagnosticFiles(patchData, patch, fixedPatch, validPatch, success);
+            if (success) {
+                successCount++;
+            }
+            if (updatedContent === patchData.after ||
+                fixedPatch === patchData.patch) {
+                console.log("SUCCESS");
+                console.log("EXACT MATCH");
+            }
+            else {
+                console.log("AI SUGGESTED");
+                console.log(patch);
+                console.log("FIXED PATCH");
+                console.log(fixedPatch);
+                console.log("ACTUAL");
+                console.log(patchData.patch);
+                console.log("DID PATCH APPLY?", !!updatedContent);
+            }
+            if (!success) {
+            }
+            const invalidPatch = (0, patch_1.hunksToPatch)(invalidHunks);
+            const invalidHunksMessage = invalidHunks.length
+                ? `Patch Partially Applied: \n Invalid Hunks: \n${invalidPatch} `
+                : "";
+            const appliedMessage = validHunks.length
+                ? `Valid Hunks Applied: \n${validPatch}`
+                : "";
+            return `${invalidHunksMessage} \n ${appliedMessage}`;
+        }
+        patchAgent.tools.setFunction("patchFile", testPatchFile);
+        const parsedAnswer = (0, diff_1.parsePatch)(patchData.patch);
+        const fileName = parsedAnswer[0].oldFileName;
+        await patchAgent.call(`Can you modify this file:
+file path: ${fileName}
+${patchData.before}
+
+      so that it would look like this:
+${patchData.after}?`);
+        console.log({ successCount, attempts, total: dataset.length });
+    }
+}
+async function saveToolDiagnosticFiles(patchData, patch, fixedPatch, validPatch, success) {
+    const failureDir = success ? "success" : "fail";
+    const patchHash = await (0, hashes_1.md5Hash)(patchData.patch);
+    const dirName = `./src/dataset/diffs/test_files/${failureDir}/${patchHash}`;
+    await (0, utils_1.mkdir)(dirName, { recursive: true });
+    await (0, utils_1.writeFile)(`${dirName}/before.ignore.ts`, patchData.before);
+    await (0, utils_1.writeFile)(`${dirName}/ai-patch.diff`, patch);
+    await (0, utils_1.writeFile)(`${dirName}/actual-patch.diff`, patchData.patch);
+    await (0, utils_1.writeFile)(`${dirName}/after.ignore.ts`, patchData.after);
+    await (0, utils_1.writeFile)(`${dirName}/ai-fixed-patch.diff`, fixedPatch);
+    await (0, utils_1.writeFile)(`${dirName}/ai-valid-patch.diff`, validPatch);
+}
+if (require.main === module) {
+    testDataset();
+}
+//# sourceMappingURL=test.js.map

package/ts_build/src/dataset/diffs/test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test.js","sourceRoot":"","sources":["../../../../src/dataset/diffs/test.ts"],"names":[],"mappings":";;AAAA,uCAAqC;AAOrC,+BAAyD;AACzD,0DAA6D;AAC7D,uCAAoD;AACpD,oDAKkC;AAClC,yCAAuC;AACvC,MAAM,OAAO,GAAG,EAAE,CAAC;AAEnB,MAAM,cAAe,SAAQ,uBAAa;IACxC,IAAI,GAAG,gBAAgB,CAAC;IAExB,KAAK,CAAC,kBAAkB,CAAC,SAAiB;QACxC,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAE/D,YAAY,CAAC,IAAI,CAAC;YAChB,OAAO,EACL,6GAA6G;YAC/G,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;QAEH,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,QAAuC;QAC/D,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAE/D,YAAY,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;QACH,OAAO,YAAY,CAAC;IACtB,CAAC;CACF;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,UAAU,GAAG,IAAI,cAAc,EAAE,CAAC;IAExC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,QAAQ,GAAG,EAAE,CAAC;IAEpB,MAAM,QAAQ,GAAG,OAAO;SACrB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;SAChD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;SAC/B,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAE7B,KAAK,MAAM,SAAS,IAAI,QAAQ,EAAE;QAChC,MAAM,IAAI,GAAG,MAAM,IAAA,gBAAO,EAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAE5C,IAAI,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE;YAEjC,SAAS;SACV;QAED,MAAM,OAAO,GAAG,IAAA,kBAAU,EAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,YAAY,GAAG,OAAO;aACzB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;aAC/D,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAG9B,IAAI,YAAY,GAAG,EAAE,EAAE;YACrB,SAAS;SACV;QAED,KAAK,UAAU,aAAa,CAC1B,QAAgB,EAChB,KAAa;YAKb,QAAQ,EAAE,CAAC;YAEX,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC;YACzC,MAAM,UAAU,GAAG,IAAA,gBAAQ,EAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACrD,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,IAAA,uBAAe,EAClD,eAAe,EACf,KAAK,CACN,CAAC;YACF,MAAM,UAAU,GAAG,IAAA,oBAAY,EAAC,UAAU,CAAC,CAAC;YAE5C,MAAM,cAAc,GAAG,IAAA,iBAAU,EAAC,eAAe,EAAE,UAAU,CAAC,CAAC;YAC/D,MAAM,OAAO,GAAG,CAAC,CAAC,cAAc,IAAI,UAAU,CAAC,MAAM,CAAC;YACtD,MAAM,uBAAuB,CAC3B,SAAS,EACT,KAAK,EACL,UAAU,EACV,UAAU,EACV,OAAO,CACR,CAAC;YAEF,IAAI,OAAO,EAAE;gBACX,YAAY,EAAE,CAAC;aAChB;YAED,IACE,cAAc,KAAK,SAAS,CAAC,KAAK;gBAClC,UAAU,KAAK,SAAS,CAAC,KAAK,EAC9B;gBACA,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;aAC5B;iBAAM;gBACL,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAEnB,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;gBAC3B,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAExB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBAE7B,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC;aACnD;YAED,IAAI,CAAC,OAAO,EAAE;aAEb;YAED,MAAM,YAAY,GAAG,IAAA,oBAAY,EAAC,YAAY,CAAC,CAAC;YAChD,MAAM,mBAAmB,GAAG,YAAY,CAAC,MAAM;gBAC7C,CAAC,CAAC,gDAAgD,YAAY,GAAG;gBACjE,CAAC,CAAC,EAAE,CAAC;YAEP,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM;gBACtC,CAAC,CAAC,0BAA0B,UAAU,EAAE;gBACxC,CAAC,CAAC,EAAE,CAAC;YAEP,OAAO,GAAG,mBAAmB,OAAO,cAAc,EAAE,CAAC;QACvD,CAAC;QAED,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;QAEzD,MAAM,YAAY,GAAG,IAAA,iBAAU,EAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;QAE7C,MAAM,UAAU,CAAC,IAAI,CAAC;aACb,QAAQ;EACnB,SAAS,CAAC,MAAM;;;EAGhB,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;QAelB,OAAO,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;KAChE;AACH,CAAC;AAED,KAAK,UAAU,uBAAuB,CACpC,SAA4B,EAC5B,KAAa,EACb,UAAkB,EAClB,UAAkB,EAClB,OAAO;IAEP,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,IAAA,gBAAO,EAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,kCAAkC,UAAU,IAAI,SAAS,EAAE,CAAC;IAC5E,MAAM,IAAA,aAAK,EAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1C,MAAM,IAAA,iBAAS,EAAC,GAAG,OAAO,mBAAmB,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACjE,MAAM,IAAA,iBAAS,EAAC,GAAG,OAAO,gBAAgB,EAAE,KAAK,CAAC,CAAC;IACnD,MAAM,IAAA,iBAAS,EAAC,GAAG,OAAO,oBAAoB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;IACjE,MAAM,IAAA,iBAAS,EAAC,GAAG,OAAO,kBAAkB,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,IAAA,iBAAS,EAAC,GAAG,OAAO,sBAAsB,EAAE,UAAU,CAAC,CAAC;IAC9D,MAAM,IAAA,iBAAS,EAAC,GAAG,OAAO,sBAAsB,EAAE,UAAU,CAAC,CAAC;AAChE,CAAC;AAED,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE;IAC3B,WAAW,EAAE,CAAC;CACf"}

package/ts_build/src/embeddings.d.ts

@@ -0,0 +1,19 @@
+import { Config, Embeddable, EmbeddingBase } from "./types";
+import { cosineSimilarity } from "./utils";
+export { cosineSimilarity };
+export declare function loadEmbedding(filePath: string): Promise<Embeddable<any>[]>;
+export declare function getConfiguredEmbeddingMap(): Promise<{
+    [filePath: string]: Embeddable<any>[];
+}>;
+export declare function getConfiguredEmbeddings(): Promise<Embeddable<any>[]>;
+export declare function embedSource(model: Config["embeddingModel"], source: Config["embedSources"][0], ignorePattern: string[]): Promise<void>;
+export declare function embed(model: string, id: string, text: string, metadata: any, embeddings: Embeddable[], prompt?: string, chunkSize?: number, uploadMode?: boolean, minLength?: number): Promise<string[]>;
+export declare function isEmbeddingFile(inputFile: string): Promise<boolean>;
+export declare function embedJson(model: string, inputFile: string, source: Config["embedSources"][0]): Promise<void>;
+export declare function embedKind(model: Config["embeddingModel"], id: string, source: Config["embedSources"][0], embeddings?: Embeddable<any>[], save?: boolean): Promise<void>;
+export declare function handleAllKinds(id: string, source: Config["embedSources"][0]): Promise<Partial<Embeddable>[]>;
+export declare function saveEmbedding(output: string, embeddings: Embeddable[]): Promise<void>;
+export declare function pruneEmbedding(id: string, chunkIds: string[], embeddings: Embeddable[]): Embeddable<any>[];
+export declare function pruneVector(embeddings: Embeddable[]): void;
+export declare function pruneMetadata(embeddings: Embeddable[], characterLimit?: number): void;
+export declare function queryEmbedding<E>(query: string, embeddings: Embeddable<E>[], model?: string): Promise<EmbeddingBase<E>[]>;

package/ts_build/src/embeddings.js

@@ -0,0 +1,322 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queryEmbedding = exports.pruneMetadata = exports.pruneVector = exports.pruneEmbedding = exports.saveEmbedding = exports.handleAllKinds = exports.embedKind = exports.embedJson = exports.isEmbeddingFile = exports.embed = exports.embedSource = exports.getConfiguredEmbeddings = exports.getConfiguredEmbeddingMap = exports.loadEmbedding = exports.cosineSimilarity = void 0;
+const glob_1 = __importDefault(require("glob"));
+const path = __importStar(require("path"));
+const config_1 = require("./config");
+const types_1 = require("./types");
+const utils_1 = require("./utils");
+Object.defineProperty(exports, "cosineSimilarity", { enumerable: true, get: function () { return utils_1.cosineSimilarity; } });
+const ai_1 = require("./ai");
+const plugins_1 = require("./plugins/plugins");
+const hashes_1 = require("./hashes");
+const conversion_1 = require("./conversion");
+const clients_1 = require("./clients");
+async function loadEmbedding(filePath) {
+    if (await (0, utils_1.fileExists)(filePath)) {
+        return JSON.parse(await (0, utils_1.readFile)(filePath, "utf8"));
+    }
+    return [];
+}
+exports.loadEmbedding = loadEmbedding;
+async function getConfiguredEmbeddingMap() {
+    const config = await (0, config_1.getConfig)();
+    const files = Array.from(new Set(config.embedSources.map((s) => s.output)));
+    const embeddings = {};
+    for (const file of files) {
+        if (!embeddings[file]) {
+            embeddings[file] = [];
+        }
+        const fileEmbeddings = await loadEmbedding(file);
+        embeddings[file].push(...fileEmbeddings);
+    }
+    return embeddings;
+}
+exports.getConfiguredEmbeddingMap = getConfiguredEmbeddingMap;
+async function getConfiguredEmbeddings() {
+    const config = await (0, config_1.getConfig)();
+    const files = Array.from(new Set(config.embedSources.map((s) => s.output)));
+    const embeddings = [];
+    for (const file of files) {
+        const fileEmbeddings = await loadEmbedding(file);
+        embeddings.push(...fileEmbeddings);
+    }
+    return embeddings;
+}
+exports.getConfiguredEmbeddings = getConfiguredEmbeddings;
+function getChunkId(id, index, chunkSize) {
+    const split = id.split("-");
+    if (split.length > 1 &&
+        Number.isInteger(parseInt(split[split.length - 1], 10))) {
+        return id;
+    }
+    return chunkSize ? `${id}-${index}` : id;
+}
+async function embedSource(model, source, ignorePattern) {
+    if (!source.input) {
+        console.log("Skipping", source.output, "with blank input property");
+        return;
+    }
+    console.log("Embedding", source.input, "to", source.output);
+    let files = await glob_1.default.sync(source.input, { ignore: ignorePattern });
+    if (source.kind && files.length === 0) {
+        files = [source.input];
+    }
+    console.log(`Found ${files.length} files`);
+    if (files.length > 100) {
+        console.error("woah there, that's a lot of files. I'm not going to embed that many");
+    }
+    console.log(files);
+    const embeddings = await loadEmbedding(source.output);
+    let batch = [];
+    let index = 0;
+    for (const file of files) {
+        index++;
+        const shouldSave = batch.length > 20 || index === files.length;
+        if (shouldSave) {
+            await Promise.all(batch);
+            batch = [];
+        }
+        batch.push(embedKind(model, file, source, embeddings, shouldSave));
+    }
+    if (batch.length > 0) {
+        await Promise.all(batch);
+    }
+    await saveEmbedding(source.output, embeddings);
+}
+exports.embedSource = embedSource;
+async function embed(model, id, text, metadata, embeddings, prompt, chunkSize, uploadMode, minLength) {
+    let chunks = [text];
+    if (!text) {
+        console.log("Skipping", id, "with blank text");
+        return [];
+    }
+    if (chunkSize) {
+        chunks = await (0, ai_1.chunkText)(text, chunkSize);
+    }
+    const MAX_CHUNKS = 300;
+    if (chunks.length <= MAX_CHUNKS) {
+        chunks = chunks.slice(0, MAX_CHUNKS);
+    }
+    const dontPrune = [];
+    const updates = new Array();
+    for (let index = 0; index < chunks.length; index++) {
+        const chunkId = getChunkId(id, index, chunkSize);
+        let textOfChunk = chunks[index];
+        const tooShort = minLength && textOfChunk.length < minLength;
+        if (tooShort) {
+            console.log("Skipping (too short)", chunkId);
+            continue;
+        }
+        dontPrune.push(chunkId);
+        const alreadyEmbedded = embeddings.find((e) => e.id === chunkId && e.text === textOfChunk);
+        if (alreadyEmbedded) {
+            console.log("Skipping", chunkId);
+            continue;
+        }
+        if (prompt) {
+            console.log("Summarizing", textOfChunk);
+            textOfChunk = await (0, ai_1.summarizeTexts)([textOfChunk], prompt);
+        }
+        let vector = [];
+        if (!uploadMode) {
+            console.log("Embedding", chunkId);
+            const providerEmbeddings = await clients_1.Clients.createEmbedding("", {
+                input: textOfChunk,
+                model: model || types_1.Models.openai.EmbeddingAda2,
+            });
+            vector = providerEmbeddings.data[0].embedding;
+        }
+        const embeddable = {
+            id: chunkId,
+            text: textOfChunk,
+            vector,
+            metadata: {
+                ...metadata,
+                ...(prompt && { text: ai_1.chunkText }),
+            },
+        };
+        embeddings.push(embeddable);
+        updates.push(chunkId);
+    }
+    pruneEmbedding(id, dontPrune, embeddings);
+    return updates;
+}
+exports.embed = embed;
+async function isEmbeddingFile(inputFile) {
+    const filePath = path.join(__dirname, inputFile);
+    const sourceJson = JSON.parse(await (0, utils_1.readFile)(inputFile, "utf8"));
+    const isEmbedding = Array.isArray(sourceJson) &&
+        sourceJson.every((e) => e.id && e.text && e.metadata);
+    console.log(`Checking file ${inputFile} for embeddings: ${isEmbedding}`);
+    return isEmbedding;
+}
+exports.isEmbeddingFile = isEmbeddingFile;
+async function embedJson(model, inputFile, source) {
+    const { prompt, output, uploadMode, chunkSize } = source;
+    const filePath = path.join(__dirname, inputFile);
+    const sourceJson = JSON.parse(await (0, utils_1.readFile)(inputFile, "utf8"));
+    const embeddings = await loadEmbedding(output);
+    let updates = [];
+    let batch = [];
+    for (const row of sourceJson) {
+        if (embeddings.find((e) => e.id === row.id)) {
+            continue;
+        }
+        console.log("Embedding", row.id);
+        batch.push(embed(model, row.id, row.text, row.metadata, embeddings, prompt, chunkSize, uploadMode, source.minLength));
+        let embedded = [];
+        if (batch.length > 20) {
+            embedded = (await Promise.all(batch)).flat();
+            batch = [];
+        }
+        updates.push(...embedded);
+        if (updates.length > 20) {
+            await saveEmbedding(output, embeddings);
+            updates = [];
+        }
+    }
+    await Promise.all(batch);
+    await saveEmbedding(output, embeddings);
+}
+exports.embedJson = embedJson;
+async function embedKind(model, id, source, embeddings = [], save = true) {
+    const { prompt, output, uploadMode, chunkSize } = source;
+    const stat = !source.kind && (await (0, utils_1.fileStat)(id));
+    if (stat && stat.isDirectory()) {
+        return;
+    }
+    if (id.endsWith(".json") && (await isEmbeddingFile(id))) {
+        console.log("Embedding JSON", id);
+        return embedJson(model, id, source);
+    }
+    const toEmbed = await handleAllKinds(id, source);
+    const updates = [];
+    for (const row of toEmbed) {
+        const { id: rowId, text, metadata } = row;
+        const embedded = await embed(model, rowId, text, metadata, embeddings, prompt, chunkSize, uploadMode);
+        updates.push(...embedded);
+    }
+    if (save && updates.length > 0) {
+        await saveEmbedding(output, embeddings);
+    }
+}
+exports.embedKind = embedKind;
+async function handleFileKind(filePath) {
+    return [
+        {
+            id: filePath,
+            text: await (0, conversion_1.convertToText)(filePath),
+            metadata: {
+                filepath: filePath,
+                date: new Date().toISOString(),
+            },
+        },
+    ];
+}
+async function handleTextKind(contents) {
+    const hash = await (0, hashes_1.md5Hash)(contents);
+    return [
+        {
+            id: hash,
+            text: contents,
+            metadata: {
+                date: new Date().toISOString(),
+            },
+        },
+    ];
+}
+async function handleAllKinds(id, source) {
+    const { input, kind } = source;
+    const contents = "";
+    const ids = [];
+    if (plugins_1.Plugins.isPlugin(kind)) {
+        console.log("Embedding with plugin", kind);
+        return plugins_1.Plugins.embed(kind, input);
+    }
+    switch (kind) {
+        case "text":
+            return handleTextKind(source.input);
+        case "file":
+        default:
+            return handleFileKind(id);
+    }
+}
+exports.handleAllKinds = handleAllKinds;
+async function saveEmbedding(output, embeddings) {
+    const fileString = "[" + embeddings.map((e) => JSON.stringify(e)).join(",") + "]";
+    await (0, utils_1.writeFile)(output, fileString);
+}
+exports.saveEmbedding = saveEmbedding;
+function pruneEmbedding(id, chunkIds, embeddings) {
+    const relatedChunks = embeddings.filter((e) => e.id.startsWith(id));
+    for (const chunk of relatedChunks) {
+        if (!chunkIds.includes(chunk.id)) {
+            console.log("Removing", chunk.id);
+            const index = embeddings.findIndex((e) => e.id === chunk.id);
+            embeddings.splice(index, 1);
+        }
+    }
+    return embeddings;
+}
+exports.pruneEmbedding = pruneEmbedding;
+function pruneVector(embeddings) {
+    for (const entry of embeddings) {
+        delete entry.vector;
+    }
+}
+exports.pruneVector = pruneVector;
+function pruneMetadata(embeddings, characterLimit = 5000) {
+    for (const entry of embeddings) {
+        for (const key of Object.keys(entry.metadata)) {
+            if (JSON.stringify(entry.metadata[key]).length > characterLimit) {
+                delete entry.metadata[key];
+            }
+        }
+    }
+}
+exports.pruneMetadata = pruneMetadata;
+async function queryEmbedding(query, embeddings, model = types_1.Models.openai.EmbeddingAda2) {
+    const providerEmbeddings = await clients_1.Clients.createEmbedding("", {
+        input: (0, utils_1.takeFirstNWords)(query, 5000),
+        model,
+    });
+    const queryVector = providerEmbeddings.data[0].embedding;
+    const results = new Array();
+    for (const embedding of embeddings) {
+        const similarity = (0, utils_1.cosineSimilarity)(embedding.vector, queryVector);
+        results.push({
+            ...embedding,
+            similarity,
+        });
+    }
+    return results.sort((a, b) => b.similarity - a.similarity);
+}
+exports.queryEmbedding = queryEmbedding;
+//# sourceMappingURL=embeddings.js.map

package/ts_build/src/embeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/embeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AACxB,2CAA6B;AAC7B,qCAAiD;AACjD,mCAA4E;AAC5E,mCAOiB;AAOR,iGATP,wBAAgB,OASO;AANzB,6BAAyD;AACzD,+CAA4C;AAC5C,qCAAmC;AACnC,6CAA6C;AAC7C,uCAAoC;AAI7B,KAAK,UAAU,aAAa,CAAC,QAAgB;IAClD,IAAI,MAAM,IAAA,kBAAU,EAAC,QAAQ,CAAC,EAAE;QAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,IAAA,gBAAQ,EAAC,QAAQ,EAAE,MAAM,CAAC,CAAiB,CAAC;KACrE;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AALD,sCAKC;AAEM,KAAK,UAAU,yBAAyB;IAC7C,MAAM,MAAM,GAAG,MAAM,IAAA,kBAAS,GAAE,CAAC;IACjC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAyC,EAAE,CAAC;IAC5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;YACrB,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;SACvB;QAED,MAAM,cAAc,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QACjD,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;KAC1C;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAbD,8DAaC;AAEM,KAAK,UAAU,uBAAuB;IAC3C,MAAM,MAAM,GAAG,MAAM,IAAA,kBAAS,GAAE,CAAC;IACjC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAiB,EAAE,CAAC;IACpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,MAAM,cAAc,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QACjD,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC;KACpC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AATD,0DASC;AAED,SAAS,UAAU,CAAC,EAAU,EAAE,KAAa,EAAE,SAAiB;IAC9D,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC5B,IACE,KAAK,CAAC,MAAM,GAAG,CAAC;QAChB,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EACvD;QAEA,OAAO,EAAE,CAAC;KACX;IAED,OAAO,SAAS,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC3C,CAAC;AAEM,KAAK,UAAU,WAAW,CAC/B,KAA+B,EAC/B,MAAiC,EACjC,aAAuB;IAEvB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;QACjB,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,2BAA2B,CAAC,CAAC;QACpE,OAAO;KACR;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5D,IAAI,KAAK,GAAG,MAAM,cAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;IAErE,IAAI,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACrC,KAAK,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;KACxB;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;IAC3C,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE;QACtB,OAAO,CAAC,KAAK,CACX,qEAAqE,CACtE,CAAC;KACH;IACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,UAAU,GAAiB,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpE,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,KAAK,EAAE,CAAC;QACR,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,EAAE,IAAI,KAAK,KAAK,KAAK,CAAC,MAAM,CAAC;QAC/D,IAAI,UAAU,EAAE;YACd,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACzB,KAAK,GAAG,EAAE,CAAC;SACZ;QACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;KACpE;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;QACpB,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KAC1B;IAGD,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;AACjD,CAAC;AA1CD,kCA0CC;AAEM,KAAK,UAAU,KAAK,CACzB,KAAa,EACb,EAAU,EACV,IAAY,EACZ,QAAa,EACb,UAAwB,EACxB,MAAe,EACf,SAAkB,EAClB,UAAoB,EACpB,SAAkB;IAElB,IAAI,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;IAEpB,IAAI,CAAC,IAAI,EAAE;QACT,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,EAAE,iBAAiB,CAAC,CAAC;QAC/C,OAAO,EAAE,CAAC;KACX;IAED,IAAI,SAAS,EAAE;QACb,MAAM,GAAG,MAAM,IAAA,cAAS,EAAC,IAAI,EAAE,SAAS,CAAC,CAAC;KAC3C;IAED,MAAM,UAAU,GAAG,GAAG,CAAC;IAEvB,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE;QAE/B,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;KACtC;IAED,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,MAAM,OAAO,GAAG,IAAI,KAAK,EAAU,CAAC;IACpC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,IAAI,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAEhC,MAAM,QAAQ,GAAG,SAAS,IAAI,WAAW,CAAC,MAAM,GAAG,SAAS,CAAC;QAC7D,IAAI,QAAQ,EAAE;YACZ,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,OAAO,CAAC,CAAC;YAC7C,SAAS;SACV;QAED,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxB,MAAM,eAAe,GAAG,UAAU,CAAC,IAAI,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CAClD,CAAC;QACF,IAAI,eAAe,EAAE;YACnB,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACjC,SAAS;SACV;QAED,IAAI,MAAM,EAAE;YACV,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YACxC,WAAW,GAAG,MAAM,IAAA,mBAAc,EAAC,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC;SAC3D;QAED,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,UAAU,EAAE;YACf,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAClC,MAAM,kBAAkB,GAAG,MAAM,iBAAO,CAAC,eAAe,CAAC,EAAE,EAAE;gBAC3D,KAAK,EAAE,WAAW;gBAClB,KAAK,EAAE,KAAK,IAAI,cAAM,CAAC,MAAM,CAAC,aAAa;aAC5C,CAAC,CAAC;YAEH,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;SAC/C;QAED,MAAM,UAAU,GAAe;YAC7B,EAAE,EAAE,OAAO;YACX,IAAI,EAAE,WAAW;YACjB,MAAM;YACN,QAAQ,EAAE;gBACR,GAAG,QAAQ;gBACX,GAAG,CAAC,MAAM,IAAI,EAAE,IAAI,EAAE,cAAS,EAAE,CAAC;aACnC;SACF,CAAC;QAEF,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5B,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KACvB;IAGD,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IAC1C,OAAO,OAAO,CAAC;AACjB,CAAC;AAnFD,sBAmFC;AAEM,KAAK,UAAU,eAAe,CAAC,SAAiB;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAC3B,MAAM,IAAA,gBAAQ,EAAC,SAAS,EAAE,MAAM,CAAC,CAClB,CAAC;IAElB,MAAM,WAAW,GACf,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;QACzB,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC;IAExD,OAAO,CAAC,GAAG,CAAC,iBAAiB,SAAS,oBAAoB,WAAW,EAAE,CAAC,CAAC;IACzE,OAAO,WAAW,CAAC;AACrB,CAAC;AAZD,0CAYC;AAEM,KAAK,UAAU,SAAS,CAC7B,KAAa,EACb,SAAiB,EACjB,MAAiC;IAEjC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAC3B,MAAM,IAAA,gBAAQ,EAAC,SAAS,EAAE,MAAM,CAAC,CAClB,CAAC;IAElB,MAAM,UAAU,GAAiB,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAC7D,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,KAAK,GAAG,EAAE,CAAC;IAEf,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE;QAC5B,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,EAAE;YAC3C,SAAS;SACV;QAED,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CACR,KAAK,CACH,KAAK,EACL,GAAG,CAAC,EAAE,EACN,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,QAAQ,EACZ,UAAU,EACV,MAAM,EACN,SAAS,EACT,UAAU,EACV,MAAM,CAAC,SAAS,CACjB,CACF,CAAC;QAEF,IAAI,QAAQ,GAAG,EAAE,CAAC;QAClB,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE,EAAE;YACrB,QAAQ,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7C,KAAK,GAAG,EAAE,CAAC;SACZ;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAE1B,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE;YACvB,MAAM,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YACxC,OAAO,GAAG,EAAE,CAAC;SACd;KACF;IAGD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACzB,MAAM,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;AAC1C,CAAC;AApDD,8BAoDC;AAEM,KAAK,UAAU,SAAS,CAC7B,KAA+B,EAC/B,EAAU,EACV,MAAiC,EACjC,aAAa,EAAkB,EAC/B,IAAI,GAAG,IAAI;IAEX,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;IAEzD,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,IAAA,gBAAQ,EAAC,EAAE,CAAC,CAAC,CAAC;IAClD,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;QAC9B,OAAO;KACR;IAED,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,eAAe,CAAC,EAAE,CAAC,CAAC,EAAE;QACvD,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;KACrC;IAED,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAEjD,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE;QACzB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,KAAK,EACL,KAAK,EACL,IAAI,EACJ,QAAQ,EACR,UAAU,EACV,MAAM,EACN,SAAS,EACT,UAAU,CACX,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;KAC3B;IAED,IAAI,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;QAC9B,MAAM,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;KACzC;AACH,CAAC;AAxCD,8BAwCC;AAED,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,OAAO;QACL;YACE,EAAE,EAAE,QAAQ;YACZ,IAAI,EAAE,MAAM,IAAA,0BAAa,EAAC,QAAQ,CAAC;YACnC,QAAQ,EAAE;gBACR,QAAQ,EAAE,QAAQ;gBAClB,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aAC/B;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,MAAM,IAAI,GAAG,MAAM,IAAA,gBAAO,EAAC,QAAQ,CAAC,CAAC;IACrC,OAAO;QACL;YACE,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE;gBACR,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aAC/B;SACF;KACF,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,cAAc,CAClC,EAAU,EACV,MAAiC;IAEjC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;IAC/B,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,MAAM,GAAG,GAAG,EAAE,CAAC;IAEf,IAAI,iBAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;QAC1B,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;QAC3C,OAAO,iBAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;KACnC;IACD,QAAQ,IAAI,EAAE;QACZ,KAAK,MAAM;YACT,OAAO,cAAc,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtC,KAAK,MAAM,CAAC;QACZ;YACE,OAAO,cAAc,CAAC,EAAE,CAAC,CAAC;KAC7B;AACH,CAAC;AAnBD,wCAmBC;AAEM,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,UAAwB;IAC1E,MAAM,UAAU,GACd,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACjE,MAAM,IAAA,iBAAS,EAAC,MAAM,EAAE,UAAU,CAAC,CAAC;AACtC,CAAC;AAJD,sCAIC;AAED,SAAgB,cAAc,CAC5B,EAAU,EACV,QAAkB,EAClB,UAAwB;IAExB,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE;QACjC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE;YAChC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7D,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;SAC7B;KACF;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAdD,wCAcC;AAED,SAAgB,WAAW,CAAC,UAAwB;IAClD,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;QAC9B,OAAO,KAAK,CAAC,MAAM,CAAC;KACrB;AACH,CAAC;AAJD,kCAIC;AAED,SAAgB,aAAa,CAAC,UAAwB,EAAE,cAAc,GAAG,IAAI;IAC3E,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;QAC9B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;YAE7C,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,cAAc,EAAE;gBAC/D,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;aAC5B;SACF;KACF;AACH,CAAC;AATD,sCASC;AAEM,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,UAA2B,EAC3B,KAAK,GAAG,cAAM,CAAC,MAAM,CAAC,aAAa;IAEnC,MAAM,kBAAkB,GAAG,MAAM,iBAAO,CAAC,eAAe,CAAC,EAAE,EAAE;QAC3D,KAAK,EAAE,IAAA,uBAAe,EAAC,KAAK,EAAE,IAAI,CAAC;QACnC,KAAK;KACN,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,MAAM,OAAO,GAAG,IAAI,KAAK,EAAoB,CAAC;IAC9C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;QAClC,MAAM,UAAU,GAAG,IAAA,wBAAgB,EAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,SAAS;YACZ,UAAU;SACX,CAAC,CAAC;KACJ;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;AAC7D,CAAC;AAnBD,wCAmBC"}

package/ts_build/src/hashes.d.ts

@@ -0,0 +1,6 @@
+import { Hashes } from "./types";
+export declare function getHashes(): Promise<Hashes>;
+export declare function saveHashes(hashes: any): Promise<void>;
+export declare function md5Hash(str: string): Promise<string>;
+export declare function checkNoFilesChanged(files: string[], promptHash: string, hashes: any): Promise<boolean>;
+export declare function saveAllFileHashes(files: string[], promptHash: string): Promise<void>;

package/ts_build/src/hashes.js

@@ -0,0 +1,78 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.saveAllFileHashes = exports.checkNoFilesChanged = exports.md5Hash = exports.saveHashes = exports.getHashes = void 0;
+const crypto = __importStar(require("crypto"));
+const utils_1 = require("./utils");
+const conversion_1 = require("./conversion");
+async function getHashes() {
+    const hashes = JSON.parse(await (0, utils_1.readFile)(".knowhow/.hashes.json", "utf8"));
+    return hashes;
+}
+exports.getHashes = getHashes;
+async function saveHashes(hashes) {
+    await (0, utils_1.writeFile)(".knowhow/.hashes.json", JSON.stringify(hashes, null, 2));
+}
+exports.saveHashes = saveHashes;
+async function md5Hash(str) {
+    return crypto.createHash("md5").update(str).digest("hex");
+}
+exports.md5Hash = md5Hash;
+async function checkNoFilesChanged(files, promptHash, hashes) {
+    for (const file of files) {
+        const fileContent = await (0, conversion_1.convertToText)(file);
+        const fileHash = crypto.createHash("md5").update(fileContent).digest("hex");
+        if (!hashes[file]) {
+            return false;
+        }
+        if (hashes[file].promptHash === promptHash &&
+            hashes[file].fileHash === fileHash) {
+            return true;
+        }
+        if (hashes[file][promptHash] === fileHash) {
+            return true;
+        }
+        return false;
+    }
+    return true;
+}
+exports.checkNoFilesChanged = checkNoFilesChanged;
+async function saveAllFileHashes(files, promptHash) {
+    const hashes = await getHashes();
+    for (const file of files) {
+        const fileContent = await (0, conversion_1.convertToText)(file);
+        const fileHash = crypto.createHash("md5").update(fileContent).digest("hex");
+        if (!hashes[file]) {
+            hashes[file] = {
+                fileHash,
+                promptHash,
+            };
+        }
+        hashes[file][promptHash] = fileHash;
+    }
+    await saveHashes(hashes);
+}
+exports.saveAllFileHashes = saveAllFileHashes;
+//# sourceMappingURL=hashes.js.map

package/ts_build/src/hashes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hashes.js","sourceRoot":"","sources":["../../src/hashes.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAEjC,mCAA8C;AAC9C,6CAA6C;AAEtC,KAAK,UAAU,SAAS;IAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,IAAA,gBAAQ,EAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3E,OAAO,MAAgB,CAAC;AAC1B,CAAC;AAHD,8BAGC;AAEM,KAAK,UAAU,UAAU,CAAC,MAAW;IAC1C,MAAM,IAAA,iBAAS,EAAC,uBAAuB,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAFD,gCAEC;AAEM,KAAK,UAAU,OAAO,CAAC,GAAW;IACvC,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5D,CAAC;AAFD,0BAEC;AAEM,KAAK,UAAU,mBAAmB,CACvC,KAAe,EACf,UAAkB,EAClB,MAAW;IAEX,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QAExB,MAAM,WAAW,GAAG,MAAM,IAAA,0BAAa,EAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YACjB,OAAO,KAAK,CAAC;SACd;QAED,IACE,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,KAAK,UAAU;YACtC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,EAClC;YACA,OAAO,IAAI,CAAC;SACb;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,QAAQ,EAAE;YACzC,OAAO,IAAI,CAAC;SACb;QAED,OAAO,KAAK,CAAC;KACd;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AA7BD,kDA6BC;AAEM,KAAK,UAAU,iBAAiB,CAAC,KAAe,EAAE,UAAkB;IACzE,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,MAAM,WAAW,GAAG,MAAM,IAAA,0BAAa,EAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YACjB,MAAM,CAAC,IAAI,CAAC,GAAG;gBACb,QAAQ;gBACR,UAAU;aACX,CAAC;SACH;QACD,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC;KACrC;IAED,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;AAC3B,CAAC;AAjBD,8CAiBC"}

package/ts_build/src/index.d.ts

@@ -0,0 +1,14 @@
+export * as clients from "./clients";
+export * as agents from "./agents";
+export * as ai from "./ai";
+export * as services from "./services";
+export * as embeddings from "./embeddings";
+export * as types from "./types";
+export declare function embed(): Promise<void>;
+export declare function purge(globPath: string): Promise<void>;
+export declare function upload(): Promise<void>;
+export declare function generate(): Promise<void>;
+export declare function handleMultiOutputGeneration(model: string, inputPattern: string, files: string[], prompt: string, output: string, outputExt?: string, outputName?: string, kind?: string, agent?: string): Promise<void>;
+export declare function handleSingleOutputGeneration(model: string, files: string[], prompt: string, outputFile: string, kind?: string, agent?: string): Promise<void>;
+export declare function chat(): Promise<void>;
+export declare function download(): Promise<void>;