@jpoly1219/context-extractor 0.1.2 → 0.2.1

package/dist/ocaml-type-checker.js

@@ -0,0 +1,286 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OcamlTypeChecker = void 0;
+const utils_1 = require("./utils");
+class OcamlTypeChecker {
+    getIdentifierFromDecl(typeDecl) {
+        const declRe = /(type )(.+)( =)(.*)/;
+        const match = typeDecl.match(declRe);
+        if (!match)
+            return "";
+        return match[2];
+    }
+    getTypeContextFromDecl(typeDecl) {
+        if (this.checkHole(typeDecl)) {
+            return this.checkHole(typeDecl);
+        }
+        else if (this.checkParameter(typeDecl)) {
+            return this.checkParameter(typeDecl);
+        }
+        else if (this.checkFunction(typeDecl)) {
+            return this.checkFunction(typeDecl);
+        }
+        else if (this.checkUnion(typeDecl)) {
+            return this.checkUnion(typeDecl);
+        }
+        else if (this.checkObject(typeDecl)) {
+            return this.checkObject(typeDecl);
+        }
+        else if (this.checkImports(typeDecl)) {
+            return this.checkImports(typeDecl);
+        }
+        else if (this.checkModule(typeDecl)) {
+            return this.checkModule(typeDecl);
+        }
+        else {
+            return this.checkPrimitive(typeDecl);
+        }
+    }
+    // pattern matching
+    // attempts to match strings to corresponding types, then returns an object containing the name, type span, and an interesting index
+    // base case - type can no longer be stepped into
+    // boolean, number, string, enum, unknown, any, void, null, undefined, never
+    // ideally this should be checked for before we do the for loop
+    // return typeSpan;
+    // check if hover result is from a primitive type
+    checkPrimitive(typeDecl) {
+        // type _ = boolean
+        const primitivePattern = /(type )(.+)( = )(.+)/;
+        const primitiveMatch = typeDecl.match(primitivePattern);
+        let primitiveInterestingIndex = -1;
+        if (primitiveMatch) {
+            primitiveInterestingIndex = (0, utils_1.indexOfRegexGroup)(primitiveMatch, 4);
+        }
+        if (primitiveInterestingIndex != -1) {
+            const typeName = primitiveMatch[2];
+            const typeSpan = primitiveMatch[4];
+            return { identifier: typeName, span: typeSpan, interestingIndex: primitiveInterestingIndex };
+        }
+        return null;
+    }
+    // check if hover result is from an import
+    checkImports(typeDecl) {
+        // import { _, _ };
+        const importPattern = /(import )(\{.+\})/;
+        const importMatch = typeDecl.match(importPattern);
+        let importInterestingIndex = -1;
+        if (importMatch) {
+            importInterestingIndex = (0, utils_1.indexOfRegexGroup)(importMatch, 2);
+        }
+        // import _;
+        const defaultImportPattern = /(import )(.+)/;
+        const defaultImportMatch = typeDecl.match(defaultImportPattern);
+        let defaultImportInterestingIndex = -1;
+        if (defaultImportMatch) {
+            defaultImportInterestingIndex = (0, utils_1.indexOfRegexGroup)(defaultImportMatch, 2);
+        }
+        if (importInterestingIndex != -1) {
+            const typeName = importMatch[2];
+            const typeSpan = importMatch[2];
+            return { identifier: typeName, span: typeSpan, interestingIndex: importInterestingIndex };
+        }
+        else if (defaultImportInterestingIndex != -1) {
+            const typeName = defaultImportMatch[2];
+            const typeSpan = defaultImportMatch[2];
+            return { identifier: typeName, span: typeSpan, interestingIndex: defaultImportInterestingIndex };
+        }
+        return null;
+    }
+    // check if hover result is from a module
+    checkModule(typeDecl) {
+        // module "path/to/module"
+        const modulePattern = /(module )(.+)/;
+        const moduleMatch = typeDecl.match(modulePattern);
+        let moduleInterestingIndex = -1;
+        if (moduleMatch) {
+            moduleInterestingIndex = (0, utils_1.indexOfRegexGroup)(moduleMatch, 2);
+        }
+        if (moduleInterestingIndex != -1) {
+            const typeName = moduleMatch[2];
+            const typeSpan = moduleMatch[2];
+            return { identifier: typeName, span: typeSpan, interestingIndex: moduleInterestingIndex };
+        }
+        return null;
+    }
+    // check if hover result is from an object
+    checkObject(typeDecl) {
+        // type _ = {
+        //   _: t1;
+        //   _: t2;
+        // }
+        const objectTypeDefPattern = /(type )(.+)( = )(\{.+\})/;
+        const objectTypeDefMatch = typeDecl.match(objectTypeDefPattern);
+        let objectTypeDefInterestingIndex = -1;
+        if (objectTypeDefMatch) {
+            objectTypeDefInterestingIndex = (0, utils_1.indexOfRegexGroup)(objectTypeDefMatch, 4);
+        }
+        if (objectTypeDefInterestingIndex != -1) {
+            const typeName = objectTypeDefMatch[2];
+            const typeSpan = objectTypeDefMatch[4];
+            return { identifier: typeName, span: typeSpan, interestingIndex: objectTypeDefInterestingIndex };
+        }
+        return null;
+    }
+    // check if hover result is from a union
+    checkUnion(typeDecl) {
+        // type _ = A | B | C
+        const unionPattern = /(type )(.+)( = )((.+ | )+.+)/;
+        const unionMatch = typeDecl.match(unionPattern);
+        let unionInterestingIndex = -1;
+        if (unionMatch) {
+            unionInterestingIndex = (0, utils_1.indexOfRegexGroup)(unionMatch, 4);
+        }
+        if (unionInterestingIndex != -1) {
+            const typeName = unionMatch[2];
+            const typeSpan = unionMatch[4];
+            return { identifier: typeName, span: typeSpan, interestingIndex: unionInterestingIndex };
+        }
+        return null;
+    }
+    // check if hover result is from a function
+    checkFunction(typeDecl) {
+        // const myFunc : (arg1: typ1, ...) => _
+        const es6AnnotatedFunctionPattern = /(const )(.+)(: )(\(.+\) => .+)/;
+        const es6AnnotatedFunctionMatch = typeDecl.match(es6AnnotatedFunctionPattern);
+        let es6AnnotatedFunctionInterestingIndex = -1;
+        if (es6AnnotatedFunctionMatch) {
+            es6AnnotatedFunctionInterestingIndex = (0, utils_1.indexOfRegexGroup)(es6AnnotatedFunctionMatch, 4);
+        }
+        // type _ = (_: t1) => t2
+        const es6FunctionTypeDefPattern = /(type )(.+)( = )(\(.+\) => .+)/;
+        const es6FunctionTypeDefPatternMatch = typeDecl.match(es6FunctionTypeDefPattern);
+        let es6FunctionTypeDefInterestingIndex = -1;
+        if (es6FunctionTypeDefPatternMatch) {
+            es6FunctionTypeDefInterestingIndex = (0, utils_1.indexOfRegexGroup)(es6FunctionTypeDefPatternMatch, 4);
+        }
+        // function myFunc<T>(args: types, genarg: T): returntype
+        const genericFunctionTypePattern = /(function )(.+)(\<.+\>\(.*\))(: )(.+)/;
+        const genericFunctionTypeMatch = typeDecl.match(genericFunctionTypePattern);
+        let genericFunctionTypeInterestingIndex = -1;
+        if (genericFunctionTypeMatch) {
+            genericFunctionTypeInterestingIndex = (0, utils_1.indexOfRegexGroup)(genericFunctionTypeMatch, 3);
+        }
+        // function myFunc(args: types): returntype
+        const functionTypePattern = /(function )(.+)(\(.*\))(: )(.+)/;
+        const functionTypeMatch = typeDecl.match(functionTypePattern);
+        let functionTypeInterestingIndex = -1;
+        if (functionTypeMatch) {
+            functionTypeInterestingIndex = (0, utils_1.indexOfRegexGroup)(functionTypeMatch, 3);
+        }
+        if (es6AnnotatedFunctionInterestingIndex != -1) {
+            const typeName = es6AnnotatedFunctionMatch[2];
+            const typeSpan = es6AnnotatedFunctionMatch[4];
+            return { identifier: typeName, span: typeSpan, interestingIndex: es6AnnotatedFunctionInterestingIndex };
+        }
+        else if (es6FunctionTypeDefInterestingIndex != -1) {
+            const typeName = es6FunctionTypeDefPatternMatch[2];
+            const typeSpan = es6FunctionTypeDefPatternMatch[4];
+            return { identifier: typeName, span: typeSpan, interestingIndex: es6FunctionTypeDefInterestingIndex };
+        }
+        else if (genericFunctionTypeInterestingIndex != -1) {
+            const typeName = genericFunctionTypeMatch[2];
+            const typeSpan = genericFunctionTypeMatch[3] + genericFunctionTypeMatch[4] + genericFunctionTypeMatch[5];
+            return { identifier: typeName, span: typeSpan, interestingIndex: genericFunctionTypeInterestingIndex };
+        }
+        else if (functionTypeInterestingIndex != -1) {
+            const typeName = functionTypeMatch[2];
+            const typeSpan = functionTypeMatch[3] + functionTypeMatch[4] + functionTypeMatch[5];
+            return { identifier: typeName, span: typeSpan, interestingIndex: functionTypeInterestingIndex };
+        }
+        return null;
+    }
+    // check if hover result is from a hole
+    checkHole(typeDecl) {
+        // (type parameter) T in _<T>(): T
+        const holePattern = /(\(type parameter\) T in _\<T\>\(\): T)/;
+        const match = typeDecl.match(holePattern);
+        if (match) {
+            const typeName = "hole function";
+            const typeSpan = match[1];
+            return { identifier: typeName, span: typeSpan };
+        }
+        return null;
+    }
+    // check if hover result is from a parameter
+    checkParameter(typeDecl) {
+        // (parameter) name: type
+        // const parameterPattern = /(\(parameter\) )(.+)(: )(.+))/;
+        // const parameterMatch = typeDecl.match(parameterPattern);
+        // let parameterInterestingIndex = -1;
+        // if (parameterMatch) {
+        //   parameterInterestingIndex = indexOfRegexGroup(parameterMatch, 4);
+        // }
+        //
+        // if (parameterInterestingIndex != -1) {
+        //   const typeName = parameterMatch[2];
+        //   const typeSpan = parameterMatch[4];
+        //   return { typeName: typeName, typeSpan: typeSpan, interestingIndex: parameterInterestingIndex }
+        // }
+        return null;
+    }
+    isTuple(typeSpan) {
+        return typeSpan[0] === "[" && typeSpan[typeSpan.length - 1] === "]";
+    }
+    isUnion(typeSpan) {
+        return typeSpan.includes(" | ");
+    }
+    isArray(typeSpan) {
+        return typeSpan.slice(-2) === "[]";
+    }
+    isObject(typeSpan) {
+        return typeSpan[0] === "{" && typeSpan[typeSpan.length - 1] === "}";
+    }
+    // this is a very rudimentary check, so it should be expanded upon
+    isFunction(typeSpan) {
+        return typeSpan.includes("=>");
+    }
+    isPrimitive(typeSpan) {
+        const primitives = ["string", "number", "boolean"];
+        return primitives.includes(typeSpan);
+    }
+    isTypeAlias(typeSpan) {
+        const caps = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        return caps.includes(typeSpan[0]);
+    }
+    escapeQuotes(typeSpan) {
+        return typeSpan.replace(/"/g, `\\"`);
+    }
+    parseTypeArrayString(typeStr) {
+        // Remove all spaces
+        const cleaned = typeStr.replace(/\s/g, '');
+        // Remove the outermost square brackets
+        const inner = cleaned.slice(1, -1);
+        // const inner = cleaned.slice(-1) === ";" ? cleaned.slice(1, -2) : cleaned.slice(1, -1);
+        // Split the string, respecting nested structures
+        const result = [];
+        let currentItem = '';
+        let nestLevel = 0;
+        for (const char of inner) {
+            if (char === '[')
+                nestLevel++;
+            if (char === ']')
+                nestLevel--;
+            if (char === ',' && nestLevel === 0) {
+                // check if currentItem is a name: type pair or just type
+                if (currentItem.includes(":")) {
+                    result.push(currentItem.split(":")[1]);
+                }
+                else {
+                    result.push(currentItem);
+                }
+                currentItem = '';
+            }
+            else {
+                currentItem += char;
+            }
+        }
+        if (currentItem.includes(":")) {
+            result.push(currentItem.split(":")[1]);
+        }
+        else {
+            result.push(currentItem);
+        }
+        return result;
+    }
+}
+exports.OcamlTypeChecker = OcamlTypeChecker;

package/dist/runner.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/runner.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const main_1 = require("./main");
+const types_1 = require("./types");
+// extract("/home/jacob/projects/context-extractor/targets/todo/sketch.ts").then(r => console.log("todo\n", r));
+// extract("/home/jacob/projects/context-extractor/targets/playlist/sketch.ts").then(r => console.log("playlist\n", r));
+// extract("/home/jacob/projects/context-extractor/targets/passwords/sketch.ts").then(r => console.log("passwords\n", r));
+// extract("/home/jacob/projects/context-extractor/targets/booking/sketch.ts").then(r => console.log("booking\n", r));
+// extract("/home/jacob/projects/context-extractor/targets/emojipaint/sketch.ts").then(r => console.log("emojipaint\n", r));
+(async () => {
+    try {
+        const x = await (0, main_1.extractContext)(types_1.Language.TypeScript, "/home/jacob/projects/context-extractor/targets/todo/sketch.ts", "/home/jacob/projects/context-extractor/targets/todo/");
+        console.dir(x, { depth: null });
+        // const y = await completeWithLLM(
+        //   x!,
+        //   Language.TypeScript,
+        //   "/home/jacob/projects/context-extractor/targets/todo/sketch.ts",
+        //   "/home/jacob/projects/context-extractor/credentials.json"
+        // )
+        //
+        // console.dir(y)
+        // const y = await extractContext(
+        //   Language.OCaml,
+        //   "/home/jacob/projects/context-extractor/targets/ocaml/todo/sketch.ml",
+        //   "/home/jacob/projects/context-extractor/targets/ocaml/todo/",
+        //   "/home/jacob/projects/context-extractor/config.json"
+        // );
+        // console.dir(y, { depth: null })
+    }
+    catch (err) {
+        console.log("top level err: ", err);
+    }
+    finally {
+        // Debug active handles if the app doesn't terminate
+        // if ((process as any)._getActiveHandles().length > 0) {
+        //   // console.log(`${(process as any)._getActiveHandles().length} active handles detected:`);
+        //   // console.log((process as any)._getActiveHandles());
+        //   console.log(`${(process as any)._getActiveHandles().length} active handles detected.`);
+        // }
+        // process.exit(0);
+    }
+})();
+// extractWithNew(Language.TypeScript, "/home/jacob/projects/context-extractor/targets/playlist/sketch.ts", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("playlist\n", r));
+// extractWithNew(Language.TypeScript, "/home/jacob/projects/context-extractor/targets/passwords/sketch.ts", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("passwords\n", r));
+// extractWithNew(Language.TypeScript, "/home/jacob/projects/context-extractor/targets/booking/sketch.ts", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("booking\n", r));
+// extractWithNew(Language.TypeScript, "/home/jacob/projects/context-extractor/targets/emojipaint/sketch.ts", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("emojipaint\n", r));
+// extractWithNew(Language.OCaml, "/home/jacob/projects/context-extractor/targets/ocaml/todo/sketch.ml", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("todo\n", r));
+// extractWithNew(Language.OCaml, "/home/jacob/projects/context-extractor/targets/ocaml/playlist/sketch.ml", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("playlist\n", r));
+// extractWithNew(Language.OCaml, "/home/jacob/projects/context-extractor/targets/ocaml/passwords/sketch.ml", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("passwords\n", r));
+// extractWithNew(Language.OCaml, "/home/jacob/projects/context-extractor/targets/ocaml/booking/sketch.ml", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("booking\n", r));
+// extractWithNew(Language.OCaml, "/home/jacob/projects/context-extractor/targets/ocaml/emojipaint/sketch.ml", "/home/jacob/projects/context-extractor/credentials.json").then(r => console.log("emojipaint\n", r));
+// extractWithNew(Language.TypeScript, "/app/targets/todo/sketch.ts").then(r => console.log("todo\n", r));

package/dist/types.d.ts

@@ -0,0 +1,135 @@
+import { LspClient } from "../ts-lsp-client-dist/src/lspClient";
+import { Range } from "../ts-lsp-client-dist/src/models";
+interface relevantTypeObject {
+    typeAliasDeclaration: string;
+    typeName: string;
+    typeDefinition: string;
+    typeQLClass: string;
+    components: typesObject[];
+}
+interface varsObject {
+    constDeclaration: string;
+    bindingPattern: string;
+    typeAnnotation: string;
+    init: string;
+    typeQLClass: string;
+    functionReturnType: string;
+    functionReturnTypeQLClass: string;
+    components: typesObject[];
+}
+interface typesObject {
+    typeName: string;
+    typeQLClass: string;
+}
+interface typeAndLocation {
+    typeName: string;
+    locatedFile: string;
+}
+interface relevantTypeQueryResult {
+    "#select": {
+        tuples: [{
+            label: string;
+        }, string, {
+            label: string;
+        }, string, {
+            label: string;
+        }, string][];
+    };
+}
+interface varsQueryResult {
+    "#select": {
+        tuples: [{
+            label: string;
+        }, {
+            label: string;
+        }, {
+            label: string;
+        }, {
+            label: string;
+        }, string, string, string, {
+            label: string;
+        }, string][];
+    };
+}
+interface typesQueryResult {
+    "#select": {
+        tuples: [string, string, number][];
+    };
+}
+interface typesAndLocationsQueryResult {
+    "#select": {
+        tuples: [string, string][];
+    };
+}
+interface LanguageDriver {
+    init: (lspClient: LspClient, sketchPath: string) => Promise<void>;
+    getHoleContext: (lspClient: LspClient, sketchFilePath: string) => Promise<{
+        fullHoverResult: string;
+        functionName: string;
+        functionTypeSpan: string;
+        linePosition: number;
+        characterPosition: number;
+        holeTypeDefLinePos: number;
+        holeTypeDefCharPos: number;
+        range: Range;
+        source: string;
+    }>;
+    extractRelevantTypes: (lspClient: LspClient, fullHoverResult: string, typeName: string, startLine: number, endLine: number, foundSoFar: Map<string, TypeSpanAndSourceFile>, currentFile: string) => Promise<Map<string, TypeSpanAndSourceFile>>;
+    extractRelevantHeaders: (lspClient: LspClient, sources: string[], relevantTypes: Map<string, TypeSpanAndSourceFile>, holeType: string) => Promise<Set<TypeSpanAndSourceFile>>;
+}
+type Filepath = string;
+type RelevantType = string;
+type RelevantHeader = string;
+interface Context {
+    holeType: string;
+    relevantTypes: Map<Filepath, RelevantType[]>;
+    relevantHeaders: Map<Filepath, RelevantHeader[]>;
+}
+declare enum Language {
+    TypeScript = 0,
+    OCaml = 1
+}
+interface TypeChecker {
+    getTypeContextFromDecl: (typeDecl: string) => {
+        identifier: string;
+        span: string;
+    } | null;
+}
+interface TypeSpanAndSourceFile {
+    typeSpan: string;
+    sourceFile: string;
+}
+declare enum Model {
+    None = 0,
+    GPT4 = "gpt4",
+    Starcoder2 = "starcoder2"
+}
+interface LLMConfig {
+    model: Model;
+}
+interface GPT4Config extends LLMConfig {
+    apiBase: string;
+    deployment: string;
+    gptModel: string;
+    apiVersion: string;
+    apiKey: string;
+    temperature: number;
+}
+interface GPT4PromptComponent {
+    role: string;
+    content: string;
+}
+interface TypeAnalysis {
+    kind: string;
+    text: string;
+    constituents?: TypeAnalysis[];
+    parameters?: ParameterAnalysis[];
+    returnType?: TypeAnalysis;
+    heritage?: TypeAnalysis[][];
+}
+interface ParameterAnalysis {
+    name: string;
+    optional: boolean;
+    type: TypeAnalysis;
+}
+export { relevantTypeObject, varsObject, typesObject, typeAndLocation, relevantTypeQueryResult, varsQueryResult, typesQueryResult, typesAndLocationsQueryResult, LanguageDriver, Language, TypeChecker, TypeSpanAndSourceFile, Context, Model, LLMConfig, GPT4Config, GPT4PromptComponent, TypeAnalysis, ParameterAnalysis };

package/dist/types.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Model = exports.Language = void 0;
+var Language;
+(function (Language) {
+    Language[Language["TypeScript"] = 0] = "TypeScript";
+    Language[Language["OCaml"] = 1] = "OCaml";
+})(Language || (exports.Language = Language = {}));
+var Model;
+(function (Model) {
+    Model[Model["None"] = 0] = "None";
+    Model["GPT4"] = "gpt4";
+    Model["Starcoder2"] = "starcoder2";
+})(Model || (exports.Model = Model = {}));

package/dist/typescript-driver.d.ts

@@ -0,0 +1,27 @@
+import { LspClient, Range } from "../ts-lsp-client-dist/src/main";
+import { LanguageDriver, TypeSpanAndSourceFile } from "./types";
+import { TypeScriptTypeChecker } from "./typescript-type-checker";
+export declare class TypeScriptDriver implements LanguageDriver {
+    typeChecker: TypeScriptTypeChecker;
+    init(lspClient: LspClient, sketchPath: string): Promise<void>;
+    getHoleContext(lspClient: LspClient, sketchFilePath: string): Promise<{
+        fullHoverResult: string;
+        functionName: string;
+        functionTypeSpan: string;
+        linePosition: number;
+        characterPosition: number;
+        holeTypeDefLinePos: number;
+        holeTypeDefCharPos: number;
+        range: Range;
+        source: string;
+    }>;
+    extractRelevantTypes(lspClient: LspClient, fullHoverResult: string, typeName: string, startLine: number, endLine: number, foundSoFar: Map<string, TypeSpanAndSourceFile>, // identifier -> [full hover result, source]
+    currentFile: string): Promise<Map<string, TypeSpanAndSourceFile>>;
+    extractRelevantHeaders(_: LspClient, sources: string[], relevantTypes: Map<string, TypeSpanAndSourceFile>, holeType: string): Promise<Set<TypeSpanAndSourceFile>>;
+    generateTargetTypes(relevantTypes: Map<string, TypeSpanAndSourceFile>, holeType: string): Set<string>;
+    generateTargetTypesHelper(relevantTypes: Map<string, TypeSpanAndSourceFile>, currType: string, targetTypes: Set<string>): void;
+    extractRelevantHeadersHelper(typeSpan: string, targetTypes: Set<string>, relevantTypes: Map<string, TypeSpanAndSourceFile>, relevantContext: Set<TypeSpanAndSourceFile>, line: string, source: string): void;
+    isTypeEquivalent(t1: string, t2: string, relevantTypes: Map<string, TypeSpanAndSourceFile>): boolean;
+    normalize(typeSpan: string, relevantTypes: Map<string, TypeSpanAndSourceFile>): string;
+    normalize2(typeSpan: string, relevantTypes: Map<string, TypeSpanAndSourceFile>): string;
+}