@jpoly1219/context-extractor 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,7 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.completeWithLLM = exports.extractContext = void 0;
4
+ var main_1 = require("./main");
5
+ Object.defineProperty(exports, "extractContext", { enumerable: true, get: function () { return main_1.extractContext; } });
6
+ var main_2 = require("./main");
7
+ Object.defineProperty(exports, "completeWithLLM", { enumerable: true, get: function () { return main_2.completeWithLLM; } });
package/dist/main.d.ts ADDED
@@ -0,0 +1,13 @@
1
+ import { Context, Language } from "./types";
2
+ export declare const extract: (sketchPath: string) => Promise<{
3
+ hole: string;
4
+ relevantTypes: string[];
5
+ relevantHeaders: string[];
6
+ }>;
7
+ export declare const extractWithCodeQL: (sketchPath: string) => Promise<{
8
+ hole: string;
9
+ relevantTypes: string[];
10
+ relevantHeaders: string[];
11
+ } | undefined>;
12
+ export declare const extractContext: (language: Language, sketchPath: string, repoPath: string) => Promise<Context | null>;
13
+ export declare const completeWithLLM: (ctx: Context, language: Language, sketchPath: string, configPath: string) => Promise<string>;
package/dist/main.js ADDED
@@ -0,0 +1,255 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.completeWithLLM = exports.extractContext = exports.extractWithCodeQL = exports.extract = void 0;
27
+ // import { JSONRPCEndpoint, LspClient, ClientCapabilities } from "../ts-lsp-client-dist/src/main.js"
28
+ const ts_lsp_client_1 = require("ts-lsp-client");
29
+ const child_process_1 = require("child_process");
30
+ const fs = __importStar(require("fs"));
31
+ const path = __importStar(require("path"));
32
+ const core_1 = require("./core");
33
+ const codeql_1 = require("./codeql");
34
+ const constants_1 = require("./constants");
35
+ const app_1 = require("./app");
36
+ // sketchPath: /home/<username>/path/to/sketch/dir/sketch.ts
37
+ const extract = async (sketchPath) => {
38
+ const logFile = fs.createWriteStream("log.txt");
39
+ const rootPath = path.dirname(sketchPath);
40
+ const rootUri = `file://${rootPath}`;
41
+ const sketchFileName = path.basename(sketchPath);
42
+ const workspaceFolders = [{ 'name': 'context-extractor', 'uri': rootUri }];
43
+ // initialize LS client and server
44
+ const r = (0, child_process_1.spawn)('typescript-language-server', ['--stdio']);
45
+ const e = new ts_lsp_client_1.JSONRPCEndpoint(r.stdin, r.stdout);
46
+ const c = new ts_lsp_client_1.LspClient(e);
47
+ console.log(JSON.stringify(c));
48
+ const capabilities = {
49
+ 'textDocument': {
50
+ 'codeAction': { 'dynamicRegistration': true },
51
+ 'codeLens': { 'dynamicRegistration': true },
52
+ 'colorProvider': { 'dynamicRegistration': true },
53
+ 'completion': {
54
+ 'completionItem': {
55
+ 'commitCharactersSupport': true,
56
+ 'documentationFormat': ['markdown', 'plaintext'],
57
+ 'snippetSupport': true
58
+ },
59
+ 'completionItemKind': {
60
+ 'valueSet': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
61
+ },
62
+ 'contextSupport': true,
63
+ 'dynamicRegistration': true
64
+ },
65
+ 'definition': { 'dynamicRegistration': true },
66
+ 'documentHighlight': { 'dynamicRegistration': true },
67
+ 'documentLink': { 'dynamicRegistration': true },
68
+ 'documentSymbol': {
69
+ 'dynamicRegistration': true,
70
+ 'symbolKind': {
71
+ 'valueSet': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
72
+ }
73
+ },
74
+ 'formatting': { 'dynamicRegistration': true },
75
+ 'hover': {
76
+ 'contentFormat': ['markdown', 'plaintext'],
77
+ 'dynamicRegistration': true
78
+ },
79
+ 'implementation': { 'dynamicRegistration': true },
80
+ // 'inlayhint': { 'dynamicRegistration': true },
81
+ 'onTypeFormatting': { 'dynamicRegistration': true },
82
+ 'publishDiagnostics': { 'relatedInformation': true },
83
+ 'rangeFormatting': { 'dynamicRegistration': true },
84
+ 'references': { 'dynamicRegistration': true },
85
+ 'rename': { 'dynamicRegistration': true },
86
+ 'signatureHelp': {
87
+ 'dynamicRegistration': true,
88
+ 'signatureInformation': { 'documentationFormat': ['markdown', 'plaintext'] }
89
+ },
90
+ 'synchronization': {
91
+ 'didSave': true,
92
+ 'dynamicRegistration': true,
93
+ 'willSave': true,
94
+ 'willSaveWaitUntil': true
95
+ },
96
+ 'typeDefinition': { 'dynamicRegistration': true, 'linkSupport': true },
97
+ // 'typeHierarchy': { 'dynamicRegistration': true }
98
+ },
99
+ 'workspace': {
100
+ 'applyEdit': true,
101
+ 'configuration': true,
102
+ 'didChangeConfiguration': { 'dynamicRegistration': true },
103
+ 'didChangeWatchedFiles': { 'dynamicRegistration': true },
104
+ 'executeCommand': { 'dynamicRegistration': true },
105
+ 'symbol': {
106
+ 'dynamicRegistration': true,
107
+ 'symbolKind': {
108
+ 'valueSet': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]
109
+ }
110
+ }, 'workspaceEdit': { 'documentChanges': true },
111
+ 'workspaceFolders': true
112
+ },
113
+ 'general': {
114
+ 'positionEncodings': ['utf-8']
115
+ },
116
+ };
117
+ r.stdout.on('data', (d) => logFile.write(d));
118
+ await c.initialize({
119
+ processId: process.pid,
120
+ capabilities: capabilities,
121
+ trace: 'off',
122
+ rootUri: null,
123
+ workspaceFolders: workspaceFolders,
124
+ initializationOptions: {
125
+ preferences: {
126
+ includeInlayVariableTypeHints: true
127
+ }
128
+ }
129
+ });
130
+ // inject hole function
131
+ const injectedSketchPath = `${rootPath}/injected_${sketchFileName}`;
132
+ const injectedSketchUri = `file://${injectedSketchPath}`;
133
+ const sketchFileContent = fs.readFileSync(sketchPath, "utf8");
134
+ const injectedSketchFileContent = `declare function _<T>(): T\n${sketchFileContent}`;
135
+ fs.writeFileSync(injectedSketchPath, injectedSketchFileContent);
136
+ // doucment sync client and server by notifying that the client has opened all the files inside the target directory
137
+ fs.readdirSync(rootPath).map(fileName => {
138
+ if (fs.lstatSync(`${rootPath}/${fileName}`).isFile()) {
139
+ c.didOpen({
140
+ textDocument: {
141
+ uri: `file://${rootPath}/${fileName}`,
142
+ languageId: 'typescript',
143
+ text: fs.readFileSync(`${rootPath}/${fileName}`).toString("ascii"),
144
+ version: 1
145
+ }
146
+ });
147
+ }
148
+ });
149
+ // get context of the hole
150
+ // currently only matching ES6 arrow functions
151
+ const holeContext = await (0, core_1.getHoleContext)(c, injectedSketchUri, injectedSketchFileContent);
152
+ // console.log(holeContext);
153
+ // rewrite hole function after context has been extracted to make LSP work
154
+ const trueHoleFunction = `declare function _(): ${holeContext.functionTypeSpan}`;
155
+ const trueInjectedSketchFileContent = `${trueHoleFunction}\n${sketchFileContent}`;
156
+ fs.writeFileSync(injectedSketchPath, trueInjectedSketchFileContent);
157
+ c.didChange({
158
+ textDocument: {
159
+ uri: injectedSketchUri,
160
+ version: 2
161
+ },
162
+ contentChanges: [{
163
+ text: trueInjectedSketchFileContent
164
+ }]
165
+ });
166
+ // recursively define relevant types
167
+ const outputFile = fs.createWriteStream("output.txt");
168
+ // const foundSoFar = new Map<string, string>();
169
+ const relevantTypes = await (0, core_1.extractRelevantTypes)(c, holeContext.fullHoverResult, holeContext.functionName, holeContext.functionTypeSpan, 0, "declare function _(): ".length, new Map(), injectedSketchUri, outputFile, 1);
170
+ relevantTypes.delete("_");
171
+ relevantTypes.delete("_()");
172
+ for (const [k, v] of relevantTypes.entries()) {
173
+ relevantTypes.set(k, v.slice(0, -1));
174
+ }
175
+ // console.log("relevantTypes:", relevantTypes);
176
+ logFile.end();
177
+ logFile.close();
178
+ outputFile.end();
179
+ outputFile.close();
180
+ const preludeContent = fs.readFileSync(`${rootPath}/prelude.ts`).toString("utf8");
181
+ const relevantHeaders = (0, core_1.extractRelevantHeaders)(preludeContent, relevantTypes, holeContext.functionTypeSpan);
182
+ for (const [k, v] of relevantTypes.entries()) {
183
+ relevantTypes.set(k, v + ";");
184
+ }
185
+ for (let i = 0; i < relevantHeaders.length; ++i) {
186
+ relevantHeaders[i] += ";";
187
+ }
188
+ // console.log(relevantContext);
189
+ // return { holeContext: holeContext, relevantTypes: Array.from(relevantTypes), relevantContext: relevantContext };
190
+ return { hole: holeContext.functionTypeSpan, relevantTypes: Array.from(relevantTypes, ([k, v]) => { return v; }), relevantHeaders: relevantHeaders };
191
+ };
192
+ exports.extract = extract;
193
+ const extractWithCodeQL = async (sketchPath) => {
194
+ const start = Date.now();
195
+ console.log("ROOT_DIR: ", constants_1.ROOT_DIR);
196
+ console.log("DEPS_DIR: ", constants_1.DEPS_DIR);
197
+ console.log("CODEQL_PATH: ", constants_1.CODEQL_PATH);
198
+ const targetPath = path.dirname(sketchPath);
199
+ try {
200
+ // extraction
201
+ const databasePath = (0, codeql_1.createDatabaseWithCodeQL)(constants_1.CODEQL_PATH, targetPath);
202
+ const holeType = (0, codeql_1.extractHoleType)(constants_1.CODEQL_PATH, path.join(constants_1.QUERY_DIR, "hole.ql"), databasePath, targetPath);
203
+ // console.log("holeType: ", holeType);
204
+ const relevantTypes = (0, codeql_1.extractRelevantTypesWithCodeQL)(constants_1.CODEQL_PATH, path.join(constants_1.QUERY_DIR, "relevant-types.ql"), databasePath, targetPath);
205
+ // console.log("relevantTypes: ", Array.from(relevantTypes, ([k, v]) => { return v.typeAliasDeclaration; }));
206
+ // console.log("relevantTypes: ", relevantTypes)
207
+ const headers = (0, codeql_1.extractHeadersWithCodeQL)(constants_1.CODEQL_PATH, path.join(constants_1.QUERY_DIR, "vars.ql"), databasePath, targetPath);
208
+ // console.log("headers: ", headers)
209
+ // const relevantContext = extractRelevantContextWithCodeQL(CODEQL_PATH, path.join(QUERY_DIR, "types.ql"), databasePath, targetPath, headers, relevantTypes);
210
+ // console.log("relevantContext: ", relevantContext);
211
+ // const relevantHeaders = getRelevantHeaders(CODEQL_PATH, path.join(QUERY_DIR, "types.ql"), databasePath, targetPath, headers, holeType);
212
+ // console.log("relevantHeaders: ", relevantHeaders);
213
+ const knownTypeLocations = (0, codeql_1.extractTypesAndLocations)(constants_1.CODEQL_PATH, path.join(constants_1.QUERY_DIR, "imports.ql"), databasePath, targetPath);
214
+ // console.log("known type locations: ", knownTypeLocations)
215
+ // NOTE: switch between the two header extraction methods
216
+ // const relevantHeaders = getRelevantHeaders3(CODEQL_PATH, path.join(QUERY_DIR, "types.ql"), databasePath, targetPath, headers, holeType, relevantTypes);
217
+ // console.log("relevantHeaders: ", Array.from(relevantHeaders));
218
+ const relevantHeaders = (0, codeql_1.getRelevantHeaders4)(constants_1.CODEQL_PATH, constants_1.QUERY_DIR, databasePath, targetPath, headers, holeType, relevantTypes, knownTypeLocations);
219
+ // console.log("relevantHeaders: ", Array.from(relevantHeaders));
220
+ const end = Date.now();
221
+ // console.log("end: ", end)
222
+ // console.log("elapsed: ", end - start)
223
+ return { hole: holeType.typeName, relevantTypes: Array.from(relevantTypes, ([k, v]) => { return JSON.stringify(v); }), relevantHeaders: Array.from(relevantHeaders) };
224
+ }
225
+ catch (err) {
226
+ console.error(`${targetPath}: ${err}`);
227
+ }
228
+ };
229
+ exports.extractWithCodeQL = extractWithCodeQL;
230
+ const extractContext = async (language, sketchPath, repoPath) => {
231
+ const app = new app_1.App(language, sketchPath, repoPath);
232
+ await app.run();
233
+ const res = app.getSavedResult();
234
+ app.close();
235
+ return res;
236
+ // if (!getCompletion) {
237
+ // await app.close()
238
+ // return { context: res, completion: "" };
239
+ // } else {
240
+ // if (res) {
241
+ // const completion = await app.completeWithLLM(path.dirname(sketchPath), res);
242
+ // await app.close()
243
+ // return { context: res, completion: completion };
244
+ // }
245
+ // await app.close()
246
+ // return { context: null, completion: null };
247
+ // }
248
+ };
249
+ exports.extractContext = extractContext;
250
+ const completeWithLLM = async (ctx, language, sketchPath, configPath) => {
251
+ const engine = new app_1.CompletionEngine(language, sketchPath, configPath);
252
+ const completion = await engine.completeWithLLM(ctx);
253
+ return completion;
254
+ };
255
+ exports.completeWithLLM = completeWithLLM;
@@ -0,0 +1,30 @@
1
+ import { LspClient, Range } from "ts-lsp-client";
2
+ import { LanguageDriver, TypeSpanAndSourceFile } from "./types";
3
+ import { OcamlTypeChecker } from "./ocaml-type-checker";
4
+ export declare class OcamlDriver implements LanguageDriver {
5
+ typeChecker: OcamlTypeChecker;
6
+ init(lspClient: LspClient, sketchPath: string): Promise<void>;
7
+ getHoleContext(lspClient: LspClient, sketchFilePath: string): Promise<{
8
+ fullHoverResult: string;
9
+ functionName: string;
10
+ functionTypeSpan: any;
11
+ linePosition: any;
12
+ characterPosition: any;
13
+ holeTypeDefLinePos: number;
14
+ holeTypeDefCharPos: number;
15
+ range: Range;
16
+ source: string;
17
+ }>;
18
+ extractRelevantTypes(lspClient: LspClient, fullHoverResult: string, typeName: string, startLine: number, endLine: number, foundSoFar: Map<string, TypeSpanAndSourceFile>, currentFile: string): Promise<Map<string, TypeSpanAndSourceFile>>;
19
+ extractRelevantHeaders(lspClient: LspClient, sources: string[], relevantTypes: Map<string, TypeSpanAndSourceFile>, holeType: string): Promise<Set<TypeSpanAndSourceFile>>;
20
+ extractHeaderTypeSpans(lspClient: LspClient, preludeFilePath: string): Promise<{
21
+ identifier: string;
22
+ typeSpan: string;
23
+ snippet: string;
24
+ }[]>;
25
+ generateTargetTypes(holeType: string, relevantTypes: Map<string, TypeSpanAndSourceFile>, preludeFilePath: string): Set<string>;
26
+ generateTargetTypesHelper(relevantTypes: Map<string, TypeSpanAndSourceFile>, currType: string, targetTypes: Set<string>): void;
27
+ extractRelevantHeadersHelper(typeSpan: string, targetTypes: Set<string>, relevantTypes: Map<string, TypeSpanAndSourceFile>, relevantContext: Set<TypeSpanAndSourceFile>, snippet: string, source: string): void;
28
+ isTypeEquivalent(t1: string, t2: string, relevantTypes: Map<string, TypeSpanAndSourceFile>): boolean;
29
+ normalize(typeSpan: string, relevantTypes: Map<string, TypeSpanAndSourceFile>): string | undefined;
30
+ }