@abhinav2203/coderag 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +196 -0
- package/dist/adapters/codeflow-core.d.ts +10 -0
- package/dist/adapters/codeflow-core.js +212 -0
- package/dist/adapters/codeflow-core.js.map +1 -0
- package/dist/cli.d.ts +4 -0
- package/dist/cli.js +179 -0
- package/dist/cli.js.map +1 -0
- package/dist/errors/index.d.ts +17 -0
- package/dist/errors/index.js +35 -0
- package/dist/errors/index.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer/documents.d.ts +16 -0
- package/dist/indexer/documents.js +148 -0
- package/dist/indexer/documents.js.map +1 -0
- package/dist/indexer/embedder.d.ts +7 -0
- package/dist/indexer/embedder.js +12 -0
- package/dist/indexer/embedder.js.map +1 -0
- package/dist/indexer/git-hook.d.ts +2 -0
- package/dist/indexer/git-hook.js +53 -0
- package/dist/indexer/git-hook.js.map +1 -0
- package/dist/indexer/indexer.d.ts +22 -0
- package/dist/indexer/indexer.js +108 -0
- package/dist/indexer/indexer.js.map +1 -0
- package/dist/llm/context-builder.d.ts +7 -0
- package/dist/llm/context-builder.js +93 -0
- package/dist/llm/context-builder.js.map +1 -0
- package/dist/llm/prompt.d.ts +3 -0
- package/dist/llm/prompt.js +80 -0
- package/dist/llm/prompt.js.map +1 -0
- package/dist/llm/transports.d.ts +23 -0
- package/dist/llm/transports.js +271 -0
- package/dist/llm/transports.js.map +1 -0
- package/dist/mcp/server.d.ts +10 -0
- package/dist/mcp/server.js +70 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/retrieval/page-index.d.ts +3 -0
- package/dist/retrieval/page-index.js +26 -0
- package/dist/retrieval/page-index.js.map +1 -0
- package/dist/retrieval/search.d.ts +20 -0
- package/dist/retrieval/search.js +164 -0
- package/dist/retrieval/search.js.map +1 -0
- package/dist/retrieval/traversal.d.ts +6 -0
- package/dist/retrieval/traversal.js +29 -0
- package/dist/retrieval/traversal.js.map +1 -0
- package/dist/service/coderag.d.ts +58 -0
- package/dist/service/coderag.js +208 -0
- package/dist/service/coderag.js.map +1 -0
- package/dist/service/config.d.ts +13 -0
- package/dist/service/config.js +130 -0
- package/dist/service/config.js.map +1 -0
- package/dist/service/http-metrics.d.ts +8 -0
- package/dist/service/http-metrics.js +36 -0
- package/dist/service/http-metrics.js.map +1 -0
- package/dist/service/http.d.ts +11 -0
- package/dist/service/http.js +262 -0
- package/dist/service/http.js.map +1 -0
- package/dist/store/file-cache.d.ts +6 -0
- package/dist/store/file-cache.js +24 -0
- package/dist/store/file-cache.js.map +1 -0
- package/dist/store/index-lock.d.ts +14 -0
- package/dist/store/index-lock.js +112 -0
- package/dist/store/index-lock.js.map +1 -0
- package/dist/store/manifest-store.d.ts +16 -0
- package/dist/store/manifest-store.js +49 -0
- package/dist/store/manifest-store.js.map +1 -0
- package/dist/store/vector-store.d.ts +18 -0
- package/dist/store/vector-store.js +140 -0
- package/dist/store/vector-store.js.map +1 -0
- package/dist/types.d.ts +251 -0
- package/dist/types.js +63 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/filesystem.d.ts +8 -0
- package/dist/utils/filesystem.js +38 -0
- package/dist/utils/filesystem.js.map +1 -0
- package/dist/utils/logger.d.ts +2 -0
- package/dist/utils/logger.js +19 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/text.d.ts +8 -0
- package/dist/utils/text.js +151 -0
- package/dist/utils/text.js.map +1 -0
- package/package.json +74 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;AAGxE,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC,CAAC;AAGnF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;CAC5D,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAChD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;CACjD,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACrD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IAChD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;CACrD,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAC1D,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACrC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnC,SAAS,EAAE,sBAAsB,CAAC,OAAO,CAAC,mBAAmB,CAAC;IAC9D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACnC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACpC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACrD,gBAAgB,EAAE,sBAAsB,CAAC,OAAO,CAAC,MAAM,CAAC;IACxD,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CACtD,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;IAClD,SAAS,EAAE,qBAAqB,CAAC,OAAO,CAAC;QACvC,IAAI,EAAE,CAAC;QACP,OAAO,EAAE,CAAC;QACV,eAAe,EAAE,KAAK;KACvB,CAAC;IACF,SAAS,EAAE,qBAAqB,CAAC,OAAO,CAAC;QACvC,YAAY,EAAE,CAAC;QACf,QAAQ,EAAE,CAAC;KACZ,CAAC;IACF,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC;QACnC,SAAS,EAAE,KAAK;QAChB,MAAM,EAAE,GAAG;QACX,OAAO,EAAE,MAAM;KAChB,CAAC;IACF,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC;QACnC,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,IAAI;KACX,CAAC;IACF,GAAG,EAAE,eAAe,CAAC,OAAO,CAAC;QAC3B,OAAO,EAAE,KAAK;QACd,SAAS,EAAE,mBAAmB;QAC9B,SAAS,EAAE,KAAK;QAChB,gBAAgB,EAAE,MAAM;QACxB,OAAO,EAAE,EAAE;KACZ,CAAC;IACF,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare const ensureDir: (dirPath: string) => Promise<void>;
|
|
2
|
+
export declare const fileExists: (filePath: string) => Promise<boolean>;
|
|
3
|
+
export declare const readJson: <Value>(filePath: string) => Promise<Value>;
|
|
4
|
+
export declare const writeJson: (filePath: string, value: unknown) => Promise<void>;
|
|
5
|
+
export declare const hashContent: (content: string) => string;
|
|
6
|
+
export declare const hashFile: (filePath: string) => Promise<string>;
|
|
7
|
+
export declare const resolveWithin: (basePath: string, targetPath: string) => string;
|
|
8
|
+
export declare const readTextFile: (filePath: string) => Promise<string>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
export const ensureDir = async (dirPath) => {
|
|
5
|
+
await fs.mkdir(dirPath, { recursive: true });
|
|
6
|
+
};
|
|
7
|
+
export const fileExists = async (filePath) => {
|
|
8
|
+
try {
|
|
9
|
+
await fs.access(filePath);
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return false;
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
export const readJson = async (filePath) => {
|
|
17
|
+
const content = await fs.readFile(filePath, "utf8");
|
|
18
|
+
return JSON.parse(content);
|
|
19
|
+
};
|
|
20
|
+
export const writeJson = async (filePath, value) => {
|
|
21
|
+
await ensureDir(path.dirname(filePath));
|
|
22
|
+
const tempPath = `${filePath}.tmp`;
|
|
23
|
+
await fs.writeFile(tempPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
24
|
+
await fs.rename(tempPath, filePath);
|
|
25
|
+
};
|
|
26
|
+
export const hashContent = (content) => createHash("sha256").update(content).digest("hex");
|
|
27
|
+
export const hashFile = async (filePath) => {
|
|
28
|
+
const content = await fs.readFile(filePath, "utf8");
|
|
29
|
+
return hashContent(content);
|
|
30
|
+
};
|
|
31
|
+
export const resolveWithin = (basePath, targetPath) => {
|
|
32
|
+
if (path.isAbsolute(targetPath)) {
|
|
33
|
+
return targetPath;
|
|
34
|
+
}
|
|
35
|
+
return path.resolve(basePath, targetPath);
|
|
36
|
+
};
|
|
37
|
+
export const readTextFile = async (filePath) => fs.readFile(filePath, "utf8");
|
|
38
|
+
//# sourceMappingURL=filesystem.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"filesystem.js","sourceRoot":"","sources":["../../src/utils/filesystem.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,OAAe,EAAiB,EAAE;IAChE,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC/C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,QAAgB,EAAoB,EAAE;IACrE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAS,QAAgB,EAAkB,EAAE;IACxE,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAU,CAAC;AACtC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,QAAgB,EAAE,KAAc,EAAiB,EAAE;IACjF,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,GAAG,QAAQ,MAAM,CAAC;IACnC,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC5E,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;AACtC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,OAAe,EAAU,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAE3G,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,QAAgB,EAAmB,EAAE;IAClE,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpD,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,QAAgB,EAAE,UAAkB,EAAU,EAAE;IAC5E,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAChC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,QAAgB,EAAmB,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
const log = (level, message, context) => {
|
|
2
|
+
const payload = {
|
|
3
|
+
level,
|
|
4
|
+
message,
|
|
5
|
+
...context
|
|
6
|
+
};
|
|
7
|
+
if (level === "error") {
|
|
8
|
+
console.error(JSON.stringify(payload));
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
console.log(JSON.stringify(payload));
|
|
12
|
+
};
|
|
13
|
+
export const createConsoleLogger = () => ({
|
|
14
|
+
debug: (message, context) => log("debug", message, context),
|
|
15
|
+
info: (message, context) => log("info", message, context),
|
|
16
|
+
warn: (message, context) => log("warn", message, context),
|
|
17
|
+
error: (message, context) => log("error", message, context)
|
|
18
|
+
});
|
|
19
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAEA,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,OAAe,EAAE,OAAiC,EAAE,EAAE;IAChF,MAAM,OAAO,GAAG;QACd,KAAK;QACL,OAAO;QACP,GAAG,OAAO;KACX,CAAC;IAEF,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACtB,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QACvC,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACvC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAW,EAAE,CAAC,CAAC;IAChD,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;IAC3D,IAAI,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC;IACzD,IAAI,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC;IACzD,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;CAC5D,CAAC,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare const tokenize: (text: string) => string[];
|
|
2
|
+
export declare const tokenizeMeaningfully: (text: string) => string[];
|
|
3
|
+
export declare const tokensRoughlyMatch: (left: string, right: string) => boolean;
|
|
4
|
+
export declare const embedTextDeterministically: (text: string, dimensions: number) => number[];
|
|
5
|
+
export declare const cosineSimilarity: (left: number[], right: number[]) => number;
|
|
6
|
+
export declare const lexicalOverlapScore: (query: string, candidate: string) => number;
|
|
7
|
+
export declare const weightedTokenScore: (queryTokens: string[], candidateTokens: string[]) => number;
|
|
8
|
+
export declare const uniqueNumbers: (values: number[]) => number[];
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
const TOKEN_PATTERN = /[A-Za-z0-9_]+/g;
|
|
2
|
+
const SEARCH_STOP_WORDS = new Set([
|
|
3
|
+
"a",
|
|
4
|
+
"an",
|
|
5
|
+
"and",
|
|
6
|
+
"are",
|
|
7
|
+
"at",
|
|
8
|
+
"be",
|
|
9
|
+
"by",
|
|
10
|
+
"do",
|
|
11
|
+
"does",
|
|
12
|
+
"for",
|
|
13
|
+
"from",
|
|
14
|
+
"how",
|
|
15
|
+
"in",
|
|
16
|
+
"is",
|
|
17
|
+
"it",
|
|
18
|
+
"of",
|
|
19
|
+
"on",
|
|
20
|
+
"or",
|
|
21
|
+
"the",
|
|
22
|
+
"to",
|
|
23
|
+
"what",
|
|
24
|
+
"where",
|
|
25
|
+
"which",
|
|
26
|
+
"who",
|
|
27
|
+
"why"
|
|
28
|
+
]);
|
|
29
|
+
const splitCompoundToken = (token) => token
|
|
30
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
31
|
+
.split(/[\s_]+/)
|
|
32
|
+
.map((part) => part.toLowerCase())
|
|
33
|
+
.filter(Boolean);
|
|
34
|
+
const trimSuffix = (token) => {
|
|
35
|
+
if (token.length <= 4) {
|
|
36
|
+
return token;
|
|
37
|
+
}
|
|
38
|
+
if (token.endsWith("ies")) {
|
|
39
|
+
return `${token.slice(0, -3)}y`;
|
|
40
|
+
}
|
|
41
|
+
if (token.endsWith("ing")) {
|
|
42
|
+
return token.slice(0, -3);
|
|
43
|
+
}
|
|
44
|
+
if (token.endsWith("ed")) {
|
|
45
|
+
return token.slice(0, -2);
|
|
46
|
+
}
|
|
47
|
+
if (token.endsWith("es")) {
|
|
48
|
+
return token.slice(0, -2);
|
|
49
|
+
}
|
|
50
|
+
if (token.endsWith("s")) {
|
|
51
|
+
return token.slice(0, -1);
|
|
52
|
+
}
|
|
53
|
+
return token;
|
|
54
|
+
};
|
|
55
|
+
const trimTrailingE = (token) => {
|
|
56
|
+
if (token.length <= 5 || !token.endsWith("e")) {
|
|
57
|
+
return token;
|
|
58
|
+
}
|
|
59
|
+
return token.slice(0, -1);
|
|
60
|
+
};
|
|
61
|
+
const normalizeToken = (token) => trimTrailingE(trimSuffix(token.toLowerCase()));
|
|
62
|
+
const tokenizeWith = (text, predicate) => {
|
|
63
|
+
const matches = text.match(TOKEN_PATTERN);
|
|
64
|
+
if (!matches) {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
return matches
|
|
68
|
+
.flatMap(splitCompoundToken)
|
|
69
|
+
.map(normalizeToken)
|
|
70
|
+
.filter(predicate);
|
|
71
|
+
};
|
|
72
|
+
const countPrefixMatch = (left, right) => {
|
|
73
|
+
const maxLength = Math.min(left.length, right.length);
|
|
74
|
+
let index = 0;
|
|
75
|
+
while (index < maxLength && left[index] === right[index]) {
|
|
76
|
+
index += 1;
|
|
77
|
+
}
|
|
78
|
+
return index;
|
|
79
|
+
};
|
|
80
|
+
const findMatch = (candidateTokens, queryToken) => candidateTokens.find((candidateToken) => tokensRoughlyMatch(queryToken, candidateToken));
|
|
81
|
+
export const tokenize = (text) => tokenizeWith(text, Boolean);
|
|
82
|
+
export const tokenizeMeaningfully = (text) => tokenizeWith(text, (token) => token.length > 1 && !SEARCH_STOP_WORDS.has(token));
|
|
83
|
+
export const tokensRoughlyMatch = (left, right) => {
|
|
84
|
+
if (left === right) {
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
if (Math.abs(left.length - right.length) > 1) {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
const prefixLength = countPrefixMatch(left, right);
|
|
91
|
+
return prefixLength >= 4 && prefixLength >= Math.min(left.length, right.length) - 1;
|
|
92
|
+
};
|
|
93
|
+
export const embedTextDeterministically = (text, dimensions) => {
|
|
94
|
+
const vector = new Array(dimensions).fill(0);
|
|
95
|
+
const tokens = tokenizeMeaningfully(text);
|
|
96
|
+
for (const token of tokens) {
|
|
97
|
+
const bucket = hashToken(token) % dimensions;
|
|
98
|
+
vector[bucket] = vector[bucket] + 1;
|
|
99
|
+
}
|
|
100
|
+
const magnitude = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
|
|
101
|
+
if (magnitude === 0) {
|
|
102
|
+
return vector;
|
|
103
|
+
}
|
|
104
|
+
return vector.map((value) => value / magnitude);
|
|
105
|
+
};
|
|
106
|
+
const hashToken = (token) => {
|
|
107
|
+
let value = 2166136261;
|
|
108
|
+
for (const character of token) {
|
|
109
|
+
value ^= character.charCodeAt(0);
|
|
110
|
+
value = Math.imul(value, 16777619);
|
|
111
|
+
}
|
|
112
|
+
return value >>> 0;
|
|
113
|
+
};
|
|
114
|
+
export const cosineSimilarity = (left, right) => {
|
|
115
|
+
if (left.length !== right.length) {
|
|
116
|
+
throw new Error("Cosine similarity requires vectors of equal length.");
|
|
117
|
+
}
|
|
118
|
+
let dot = 0;
|
|
119
|
+
let leftMagnitude = 0;
|
|
120
|
+
let rightMagnitude = 0;
|
|
121
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
122
|
+
const leftValue = left[index] ?? 0;
|
|
123
|
+
const rightValue = right[index] ?? 0;
|
|
124
|
+
dot += leftValue * rightValue;
|
|
125
|
+
leftMagnitude += leftValue * leftValue;
|
|
126
|
+
rightMagnitude += rightValue * rightValue;
|
|
127
|
+
}
|
|
128
|
+
if (leftMagnitude === 0 || rightMagnitude === 0) {
|
|
129
|
+
return 0;
|
|
130
|
+
}
|
|
131
|
+
return dot / (Math.sqrt(leftMagnitude) * Math.sqrt(rightMagnitude));
|
|
132
|
+
};
|
|
133
|
+
export const lexicalOverlapScore = (query, candidate) => {
|
|
134
|
+
const queryTokens = tokenizeMeaningfully(query);
|
|
135
|
+
const candidateTokens = tokenizeMeaningfully(candidate);
|
|
136
|
+
if (queryTokens.length === 0 || candidateTokens.length === 0) {
|
|
137
|
+
return 0;
|
|
138
|
+
}
|
|
139
|
+
const matchedTokens = queryTokens.filter((queryToken) => Boolean(findMatch(candidateTokens, queryToken)));
|
|
140
|
+
return matchedTokens.length / queryTokens.length;
|
|
141
|
+
};
|
|
142
|
+
export const weightedTokenScore = (queryTokens, candidateTokens) => {
|
|
143
|
+
if (queryTokens.length === 0 || candidateTokens.length === 0) {
|
|
144
|
+
return 0;
|
|
145
|
+
}
|
|
146
|
+
const uniqueQueryTokens = [...new Set(queryTokens)];
|
|
147
|
+
const matched = uniqueQueryTokens.filter((queryToken) => Boolean(findMatch(candidateTokens, queryToken)));
|
|
148
|
+
return matched.length / uniqueQueryTokens.length;
|
|
149
|
+
};
|
|
150
|
+
export const uniqueNumbers = (values) => [...new Set(values)].sort((left, right) => left - right);
|
|
151
|
+
//# sourceMappingURL=text.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"text.js","sourceRoot":"","sources":["../../src/utils/text.ts"],"names":[],"mappings":"AAAA,MAAM,aAAa,GAAG,gBAAgB,CAAC;AAEvC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,GAAG;IACH,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,MAAM;IACN,OAAO;IACP,OAAO;IACP,KAAK;IACL,KAAK;CACN,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,KAAa,EAAY,EAAE,CACrD,KAAK;KACF,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC;KACtC,KAAK,CAAC,QAAQ,CAAC;KACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;KACjC,MAAM,CAAC,OAAO,CAAC,CAAC;AAErB,MAAM,UAAU,GAAG,CAAC,KAAa,EAAU,EAAE;IAC3C,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAClC,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,KAAa,EAAU,EAAE;IAC9C,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AAEjG,MAAM,YAAY,GAAG,CAAC,IAAY,EAAE,SAAqC,EAAY,EAAE;IACrF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC1C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,OAAO;SACX,OAAO,CAAC,kBAAkB,CAAC;SAC3B,GAAG,CAAC,cAAc,CAAC;SACnB,MAAM,CAAC,SAAS,CAAC,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,IAAY,EAAE,KAAa,EAAU,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACtD,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,KAAK,GAAG,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QACzD,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,eAAyB,EAAE,UAAkB,EAAsB,EAAE,CACtF,eAAe,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC,kBAAkB,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC;AAE3F,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAY,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAY,EAAY,EAAE,CAC7D,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAEnF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAE,KAAa,EAAW,EAAE;IACzE,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACnD,OAAO,YAAY,IAAI,CAAC,IAAI,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AACtF,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,IAAY,EAAE,UAAkB,EAAY,EAAE;IACvF,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAE1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,UAAU,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IACnF,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC;AAClD,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,KAAa,EAAU,EAAE;IAC1C,IAAI,KAAK,GAAG,UAAU,CAAC;IACvB,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,KAAK,IAAI,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,KAAK,KAAK,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAAc,EAAE,KAAe,EAAU,EAAE;IAC1E,IAAI,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrC,GAAG,IAAI,SAAS,GAAG,UAAU,CAAC;QAC9B,aAAa,IAAI,SAAS,GAAG,SAAS,CAAC;QACvC,cAAc,IAAI,UAAU,GAAG,UAAU,CAAC;IAC5C,CAAC;IAED,IAAI,aAAa,KAAK,CAAC,IAAI,cAAc,KAAK,CAAC,EAAE,CAAC;QAChD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;AACtE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAU,EAAE;IAC9E,MAAM,WAAW,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,eAAe,GAAG,oBAAoB,CAAC,SAAS,CAAC,CAAC;IACxD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7D,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;AACnD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,WAAqB,EAAE,eAAyB,EAAU,EAAE;IAC7F,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7D,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO,OAAO,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC;AACnD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,MAAgB,EAAY,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@abhinav2203/coderag",
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Standalone code retrieval and MCP server for JS/TS repositories built on @abhinav2203/codeflow-core.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"main": "dist/index.js",
|
|
8
|
+
"types": "dist/index.d.ts",
|
|
9
|
+
"bin": {
|
|
10
|
+
"coderag": "dist/cli.js"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"dist"
|
|
14
|
+
],
|
|
15
|
+
"exports": {
|
|
16
|
+
".": {
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"default": "./dist/index.js"
|
|
19
|
+
},
|
|
20
|
+
"./cli": {
|
|
21
|
+
"types": "./dist/cli.d.ts",
|
|
22
|
+
"default": "./dist/cli.js"
|
|
23
|
+
},
|
|
24
|
+
"./mcp": {
|
|
25
|
+
"types": "./dist/mcp/server.d.ts",
|
|
26
|
+
"default": "./dist/mcp/server.js"
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"scripts": {
|
|
30
|
+
"build": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.json",
|
|
31
|
+
"check": "tsc --noEmit -p tsconfig.json",
|
|
32
|
+
"lint": "tsc --noEmit -p tsconfig.json",
|
|
33
|
+
"prepack": "npm run build",
|
|
34
|
+
"prepublishOnly": "npm run test && npm run build",
|
|
35
|
+
"test": "vitest run",
|
|
36
|
+
"coverage": "vitest run --coverage"
|
|
37
|
+
},
|
|
38
|
+
"keywords": [
|
|
39
|
+
"coderag",
|
|
40
|
+
"rag",
|
|
41
|
+
"mcp",
|
|
42
|
+
"typescript",
|
|
43
|
+
"javascript",
|
|
44
|
+
"code-search",
|
|
45
|
+
"repo-analysis"
|
|
46
|
+
],
|
|
47
|
+
"repository": {
|
|
48
|
+
"type": "git",
|
|
49
|
+
"url": "git+https://github.com/nehraa/CodeRag.git"
|
|
50
|
+
},
|
|
51
|
+
"homepage": "https://github.com/nehraa/CodeRag#readme",
|
|
52
|
+
"bugs": {
|
|
53
|
+
"url": "https://github.com/nehraa/CodeRag/issues"
|
|
54
|
+
},
|
|
55
|
+
"publishConfig": {
|
|
56
|
+
"access": "public"
|
|
57
|
+
},
|
|
58
|
+
"dependencies": {
|
|
59
|
+
"@abhinav2203/codeflow-core": "0.1.1",
|
|
60
|
+
"@lancedb/lancedb": "0.22.0",
|
|
61
|
+
"@modelcontextprotocol/sdk": "1.29.0",
|
|
62
|
+
"ts-morph": "27.0.2",
|
|
63
|
+
"zod": "4.3.6"
|
|
64
|
+
},
|
|
65
|
+
"devDependencies": {
|
|
66
|
+
"@types/node": "25.5.0",
|
|
67
|
+
"@vitest/coverage-v8": "4.1.0",
|
|
68
|
+
"typescript": "5.9.3",
|
|
69
|
+
"vitest": "4.1.0"
|
|
70
|
+
},
|
|
71
|
+
"engines": {
|
|
72
|
+
"node": ">=20"
|
|
73
|
+
}
|
|
74
|
+
}
|