@jpoly1219/context-extractor 0.2.7 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/src/app.d.ts +7 -4
  2. package/dist/src/app.js +319 -97
  3. package/dist/src/ast.d.ts +6 -0
  4. package/dist/src/ast.js +80 -0
  5. package/dist/src/codeql.js +17 -7
  6. package/dist/src/constants.js +17 -7
  7. package/dist/src/core.d.ts +0 -1
  8. package/dist/src/core.js +17 -7
  9. package/dist/src/main.d.ts +8 -2
  10. package/dist/src/main.js +53 -13
  11. package/dist/src/ocaml-driver.d.ts +55 -5
  12. package/dist/src/ocaml-driver.js +296 -191
  13. package/dist/src/ocaml-utils/_build/default/test_parser.bc.cjs +194658 -0
  14. package/dist/src/runner.js +143 -10
  15. package/dist/src/tree-sitter-files/queries/hole-queries/typescript.scm +36 -0
  16. package/dist/src/tree-sitter-files/queries/relevant-headers-queries/typescript-get-toplevel-headers.scm +22 -0
  17. package/dist/src/tree-sitter-files/queries/relevant-types-queries/typescript-extract-identifiers.scm +10 -0
  18. package/dist/src/tree-sitter-files/queries/relevant-types-queries/typescript-find-typedecl-given-typeidentifier.scm +11 -0
  19. package/dist/src/tree-sitter-files/wasms/tree-sitter-ocaml.wasm +0 -0
  20. package/dist/src/tree-sitter-files/wasms/tree-sitter-typescript.wasm +0 -0
  21. package/dist/src/tree-sitter-files/wasms/tree-sitter.wasm +0 -0
  22. package/dist/src/tree-sitter.d.ts +40 -0
  23. package/dist/src/tree-sitter.js +311 -0
  24. package/dist/src/types.d.ts +80 -9
  25. package/dist/src/types.js +1 -6
  26. package/dist/src/typescript-driver.d.ts +86 -12
  27. package/dist/src/typescript-driver.js +1276 -205
  28. package/dist/src/typescript-type-checker.d.ts +22 -2
  29. package/dist/src/typescript-type-checker.js +290 -9
  30. package/dist/src/utils.d.ts +11 -1
  31. package/dist/src/utils.js +87 -10
  32. package/dist/src/vscode-ide.d.ts +29 -0
  33. package/dist/src/vscode-ide.js +161 -0
  34. package/package.json +12 -6
@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
15
15
  }) : function(o, v) {
16
16
  o["default"] = v;
17
17
  });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
25
35
  Object.defineProperty(exports, "__esModule", { value: true });
26
36
  exports.EMOJIPAINT_DIR = exports.BOOKING_DIR = exports.PASSWORDS_DIR = exports.PLAYLIST_DIR = exports.TODO_DIR = exports.TARGET_DIR = exports.QUERY_DIR = exports.DEPS_DIR = exports.ROOT_DIR = exports.CODEQL_PATH = void 0;
27
37
  // import * as path from "path";
@@ -1,4 +1,3 @@
1
- /// <reference types="node" />
2
1
  import { LspClient } from "../ts-lsp-client-dist/src/main.js";
3
2
  import * as fs from "fs";
4
3
  declare const getAnnotatedFunctionHoleContext: (sketchFileContent: string) => {
package/dist/src/core.js CHANGED
@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
15
15
  }) : function(o, v) {
16
16
  o["default"] = v;
17
17
  });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
25
35
  Object.defineProperty(exports, "__esModule", { value: true });
26
36
  exports.normalize = exports.extractRelevantHeaders = exports.extractRelevantTypes = exports.getHoleContext = exports.getAnnotatedFunctionHoleContext = void 0;
27
37
  // import { LspClient, MarkupContent, Location, SymbolInformation, Range } from "ts-lsp-client"
@@ -1,4 +1,5 @@
1
- import { Context, Language } from "./types";
1
+ import { Context, IDE, Language, Position } from "./types";
2
+ import { App } from "./app";
2
3
  export declare const extract: (sketchPath: string) => Promise<{
3
4
  hole: string;
4
5
  relevantTypes: string[];
@@ -9,5 +10,10 @@ export declare const extractWithCodeQL: (sketchPath: string) => Promise<{
9
10
  relevantTypes: string[];
10
11
  relevantHeaders: string[];
11
12
  } | undefined>;
12
- export declare const extractContext: (language: Language, sketchPath: string, repoPath: string) => Promise<Context | null>;
13
+ export declare const extractContext: (language: Language, sketchPath: string, repoPath: string, ide: IDE, cursorPosition: Position) => Promise<{
14
+ res: Context | null;
15
+ elapsed: number;
16
+ }>;
17
+ export declare const spawnApp: (language: Language, sketchPath: string, repoPath: string, ide: IDE, cursorPosition: Position) => App;
18
+ export declare const extractContextWithReuse: (app: App, version: number) => Promise<Context | null>;
13
19
  export declare const completeWithLLM: (ctx: Context, language: Language, sketchPath: string, configPath: string) => Promise<string>;
package/dist/src/main.js CHANGED
@@ -15,25 +15,38 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
15
15
  }) : function(o, v) {
16
16
  o["default"] = v;
17
17
  });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
25
35
  Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.completeWithLLM = exports.extractContext = exports.extractWithCodeQL = exports.extract = void 0;
36
+ exports.completeWithLLM = exports.extractContextWithReuse = exports.spawnApp = exports.extractContext = exports.extractWithCodeQL = exports.extract = void 0;
27
37
  const main_js_1 = require("../ts-lsp-client-dist/src/main.js");
28
38
  // import { JSONRPCEndpoint, LspClient, ClientCapabilities } from "ts-lsp-client"
29
39
  // import { JSONRPCEndpoint, LspClient, ClientCapabilities } from "dist/ts-lsp-client-dist/src/main"
30
40
  const child_process_1 = require("child_process");
31
41
  const fs = __importStar(require("fs"));
32
42
  const path = __importStar(require("path"));
43
+ // import * as pprof from "pprof"
33
44
  const core_1 = require("./core");
34
45
  const codeql_1 = require("./codeql");
35
46
  const constants_1 = require("./constants");
36
47
  const app_1 = require("./app");
48
+ const utils_js_1 = require("./utils.js");
49
+ const url_1 = require("url");
37
50
  // sketchPath: /home/<username>/path/to/sketch/dir/sketch.ts
38
51
  const extract = async (sketchPath) => {
39
52
  const logFile = fs.createWriteStream("log.txt");
@@ -45,7 +58,7 @@ const extract = async (sketchPath) => {
45
58
  const r = (0, child_process_1.spawn)('typescript-language-server', ['--stdio']);
46
59
  const e = new main_js_1.JSONRPCEndpoint(r.stdin, r.stdout);
47
60
  const c = new main_js_1.LspClient(e);
48
- console.log(JSON.stringify(c));
61
+ // console.log(JSON.stringify(c));
49
62
  const capabilities = {
50
63
  'textDocument': {
51
64
  'codeAction': { 'dynamicRegistration': true },
@@ -228,12 +241,26 @@ const extractWithCodeQL = async (sketchPath) => {
228
241
  }
229
242
  };
230
243
  exports.extractWithCodeQL = extractWithCodeQL;
231
- const extractContext = async (language, sketchPath, repoPath) => {
232
- const app = new app_1.App(language, sketchPath, repoPath);
233
- await app.run();
244
+ const extractContext = async (language, sketchPath, repoPath, ide, cursorPosition) => {
245
+ // console.time("extractContext")
246
+ // const profile = await pprof.time.start(10000); // Collect for 10s
247
+ console.log("=*=*=*=");
248
+ if ((0, utils_js_1.isUri)(sketchPath)) {
249
+ sketchPath = (0, url_1.fileURLToPath)(sketchPath);
250
+ }
251
+ const start = performance.now();
252
+ const app = new app_1.App(language, sketchPath, repoPath, ide, cursorPosition);
253
+ await app.run2(1);
234
254
  const res = app.getSavedResult();
235
255
  app.close();
236
- return res;
256
+ // const buf = await pprof.encode(profile());
257
+ // fs.writeFile('wall.pb.gz', buf, (err) => {
258
+ // if (err) throw err;
259
+ // });
260
+ // console.timeEnd("extractContext")
261
+ const end = performance.now();
262
+ // console.log("elapsed:", end - start)
263
+ return { res: res, elapsed: end - start };
237
264
  // if (!getCompletion) {
238
265
  // await app.close()
239
266
  // return { context: res, completion: "" };
@@ -248,6 +275,19 @@ const extractContext = async (language, sketchPath, repoPath) => {
248
275
  // }
249
276
  };
250
277
  exports.extractContext = extractContext;
278
+ const spawnApp = (language, sketchPath, repoPath, ide, cursorPosition) => {
279
+ const app = new app_1.App(language, sketchPath, repoPath, ide, cursorPosition);
280
+ return app;
281
+ };
282
+ exports.spawnApp = spawnApp;
283
+ const extractContextWithReuse = async (app, version) => {
284
+ // await app.init();
285
+ await app.run(version);
286
+ const res = app.getSavedResult();
287
+ // app.close();
288
+ return res;
289
+ };
290
+ exports.extractContextWithReuse = extractContextWithReuse;
251
291
  const completeWithLLM = async (ctx, language, sketchPath, configPath) => {
252
292
  const engine = new app_1.CompletionEngine(language, sketchPath, configPath);
253
293
  const completion = await engine.completeWithLLM(ctx);
@@ -1,10 +1,12 @@
1
+ import * as fs from "fs";
1
2
  import { LspClient, Range } from "../ts-lsp-client-dist/src/main";
2
- import { LanguageDriver, TypeSpanAndSourceFile } from "./types";
3
+ import { LanguageDriver, TypeSpanAndSourceFile, TypeSpanAndSourceFileAndAst } from "./types";
3
4
  import { OcamlTypeChecker } from "./ocaml-type-checker";
4
5
  export declare class OcamlDriver implements LanguageDriver {
5
6
  typeChecker: OcamlTypeChecker;
6
- init(lspClient: LspClient, sketchPath: string): Promise<void>;
7
- getHoleContext(lspClient: LspClient, sketchFilePath: string): Promise<{
7
+ init(lspClient: LspClient | null, sketchPath: string): Promise<void>;
8
+ injectHole(sketchPath: string): void;
9
+ getHoleContext(lspClient: LspClient | null, sketchFilePath: string): Promise<{
8
10
  fullHoverResult: string;
9
11
  functionName: string;
10
12
  functionTypeSpan: any;
@@ -15,8 +17,55 @@ export declare class OcamlDriver implements LanguageDriver {
15
17
  range: Range;
16
18
  source: string;
17
19
  }>;
18
- extractRelevantTypes(lspClient: LspClient, fullHoverResult: string, typeName: string, startLine: number, endLine: number, foundSoFar: Map<string, TypeSpanAndSourceFile>, currentFile: string): Promise<Map<string, TypeSpanAndSourceFile>>;
19
- extractRelevantHeaders(lspClient: LspClient, sources: string[], relevantTypes: Map<string, TypeSpanAndSourceFile>, holeType: string): Promise<Set<TypeSpanAndSourceFile>>;
20
+ getHoleContextWithCompilerAPI(sketchFilePath: string, logStream: fs.WriteStream | null): Promise<{
21
+ fullHoverResult: string;
22
+ functionName: string;
23
+ functionTypeSpan: string;
24
+ linePosition: number;
25
+ characterPosition: number;
26
+ holeTypeDefLinePos: number;
27
+ holeTypeDefCharPos: number;
28
+ range: {
29
+ start: {
30
+ line: number;
31
+ character: number;
32
+ };
33
+ end: {
34
+ line: number;
35
+ character: number;
36
+ };
37
+ };
38
+ source: string;
39
+ }>;
40
+ getHoleContextWithTreesitter(sketchFilePath: string, cursorPosition: {
41
+ line: number;
42
+ character: number;
43
+ }, logStream: fs.WriteStream | null): Promise<{
44
+ fullHoverResult: string;
45
+ functionName: string;
46
+ functionTypeSpan: string;
47
+ linePosition: number;
48
+ characterPosition: number;
49
+ holeTypeDefLinePos: number;
50
+ holeTypeDefCharPos: number;
51
+ range: {
52
+ start: {
53
+ line: number;
54
+ character: number;
55
+ };
56
+ end: {
57
+ line: number;
58
+ character: number;
59
+ };
60
+ };
61
+ source: string;
62
+ }>;
63
+ extractRelevantTypes(lspClient: LspClient | null, fullHoverResult: string, typeName: string, startLine: number, foundSoFar: Map<string, TypeSpanAndSourceFile>, currentFile: string): Promise<Map<string, TypeSpanAndSourceFile>>;
64
+ extractRelevantTypesWithCompilerAPI(fullHoverResult: string, typeName: string, linePosition: number, characterPosition: number, foundSoFar: Map<string, TypeSpanAndSourceFile>, currentFile: string): Promise<Map<string, TypeSpanAndSourceFile>>;
65
+ extractRelevantTypesWithTreesitter(lspClient: LspClient | null, fullHoverResult: string, typeName: string, startLine: number, foundSoFar: Map<string, TypeSpanAndSourceFileAndAst>, // identifier -> [full hover result, source]
66
+ currentFile: string, foundContents: Map<string, string>, // uri -> contents
67
+ logStream: fs.WriteStream | null): Promise<Map<string, TypeSpanAndSourceFileAndAst>>;
68
+ extractRelevantHeaders(lspClient: LspClient | null, sources: string[], relevantTypes: Map<string, TypeSpanAndSourceFile>, holeType: string): Promise<Set<TypeSpanAndSourceFile>>;
20
69
  extractHeaderTypeSpans(lspClient: LspClient, preludeFilePath: string): Promise<{
21
70
  identifier: string;
22
71
  typeSpan: string;
@@ -27,4 +76,5 @@ export declare class OcamlDriver implements LanguageDriver {
27
76
  extractRelevantHeadersHelper(typeSpan: string, targetTypes: Set<string>, relevantTypes: Map<string, TypeSpanAndSourceFile>, relevantContext: Set<TypeSpanAndSourceFile>, snippet: string, source: string): void;
28
77
  isTypeEquivalent(t1: string, t2: string, relevantTypes: Map<string, TypeSpanAndSourceFile>): boolean;
29
78
  normalize(typeSpan: string, relevantTypes: Map<string, TypeSpanAndSourceFile>): string | undefined;
79
+ extractRelevantHeadersWithTreesitter(lspClient: LspClient | null, sources: string[], relevantTypes: Map<string, TypeSpanAndSourceFileAndAst>, holeType: string, holeIdentifier: string, projectRoot: string): Promise<Set<TypeSpanAndSourceFile>>;
30
80
  }