@untangle-code/parser-registry 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +90 -0
- package/README.md +352 -2
- package/THIRD_PARTY_LICENSES.md +91 -0
- package/dist/boundaries/expression.d.ts +10 -0
- package/dist/boundaries/expression.d.ts.map +1 -0
- package/dist/boundaries/expression.js +95 -0
- package/dist/boundaries/expression.js.map +1 -0
- package/dist/boundaries/extract.d.ts +12 -0
- package/dist/boundaries/extract.d.ts.map +1 -0
- package/dist/boundaries/extract.js +259 -0
- package/dist/boundaries/extract.js.map +1 -0
- package/dist/boundaries/python.d.ts +6 -0
- package/dist/boundaries/python.d.ts.map +1 -0
- package/dist/boundaries/python.js +154 -0
- package/dist/boundaries/python.js.map +1 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/languages/astgrep-backend/adapter.d.ts +101 -0
- package/dist/languages/astgrep-backend/adapter.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/adapter.js +130 -0
- package/dist/languages/astgrep-backend/adapter.js.map +1 -0
- package/dist/languages/astgrep-backend/index.d.ts +29 -0
- package/dist/languages/astgrep-backend/index.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/index.js +28 -0
- package/dist/languages/astgrep-backend/index.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/bash.d.ts +38 -0
- package/dist/languages/astgrep-backend/lang/bash.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/bash.js +42 -0
- package/dist/languages/astgrep-backend/lang/bash.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/c.d.ts +33 -0
- package/dist/languages/astgrep-backend/lang/c.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/c.js +85 -0
- package/dist/languages/astgrep-backend/lang/c.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/cpp.d.ts +27 -0
- package/dist/languages/astgrep-backend/lang/cpp.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/cpp.js +38 -0
- package/dist/languages/astgrep-backend/lang/cpp.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/csharp.d.ts +24 -0
- package/dist/languages/astgrep-backend/lang/csharp.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/csharp.js +102 -0
- package/dist/languages/astgrep-backend/lang/csharp.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/index.d.ts +29 -0
- package/dist/languages/astgrep-backend/lang/index.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/index.js +22 -0
- package/dist/languages/astgrep-backend/lang/index.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/java.d.ts +26 -0
- package/dist/languages/astgrep-backend/lang/java.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/java.js +89 -0
- package/dist/languages/astgrep-backend/lang/java.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/kotlin.d.ts +44 -0
- package/dist/languages/astgrep-backend/lang/kotlin.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/kotlin.js +94 -0
- package/dist/languages/astgrep-backend/lang/kotlin.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/php.d.ts +23 -0
- package/dist/languages/astgrep-backend/lang/php.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/php.js +80 -0
- package/dist/languages/astgrep-backend/lang/php.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/ruby.d.ts +37 -0
- package/dist/languages/astgrep-backend/lang/ruby.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/ruby.js +70 -0
- package/dist/languages/astgrep-backend/lang/ruby.js.map +1 -0
- package/dist/languages/astgrep-backend/lang/swift.d.ts +34 -0
- package/dist/languages/astgrep-backend/lang/swift.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/lang/swift.js +86 -0
- package/dist/languages/astgrep-backend/lang/swift.js.map +1 -0
- package/dist/languages/astgrep-backend/languages.d.ts +55 -0
- package/dist/languages/astgrep-backend/languages.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/languages.js +214 -0
- package/dist/languages/astgrep-backend/languages.js.map +1 -0
- package/dist/languages/astgrep-backend/napi.d.ts +71 -0
- package/dist/languages/astgrep-backend/napi.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/napi.js +176 -0
- package/dist/languages/astgrep-backend/napi.js.map +1 -0
- package/dist/languages/astgrep-backend/syntax.d.ts +88 -0
- package/dist/languages/astgrep-backend/syntax.d.ts.map +1 -0
- package/dist/languages/astgrep-backend/syntax.js +180 -0
- package/dist/languages/astgrep-backend/syntax.js.map +1 -0
- package/dist/languages/index.d.ts +14 -0
- package/dist/languages/index.d.ts.map +1 -0
- package/dist/languages/index.js +37 -0
- package/dist/languages/index.js.map +1 -0
- package/dist/languages/javascript/index.d.ts +4 -0
- package/dist/languages/javascript/index.d.ts.map +1 -0
- package/dist/languages/javascript/index.js +7 -0
- package/dist/languages/javascript/index.js.map +1 -0
- package/dist/languages/jsx/index.d.ts +4 -0
- package/dist/languages/jsx/index.d.ts.map +1 -0
- package/dist/languages/jsx/index.js +7 -0
- package/dist/languages/jsx/index.js.map +1 -0
- package/dist/languages/prose/index.d.ts +53 -0
- package/dist/languages/prose/index.d.ts.map +1 -0
- package/dist/languages/prose/index.js +172 -0
- package/dist/languages/prose/index.js.map +1 -0
- package/dist/languages/prose/line-bytes.d.ts +31 -0
- package/dist/languages/prose/line-bytes.d.ts.map +1 -0
- package/dist/languages/prose/line-bytes.js +39 -0
- package/dist/languages/prose/line-bytes.js.map +1 -0
- package/dist/languages/python/create-adapter.d.ts +20 -0
- package/dist/languages/python/create-adapter.d.ts.map +1 -0
- package/dist/languages/python/create-adapter.js +9 -0
- package/dist/languages/python/create-adapter.js.map +1 -0
- package/dist/languages/python/index.d.ts +4 -0
- package/dist/languages/python/index.d.ts.map +1 -0
- package/dist/languages/python/index.js +6 -0
- package/dist/languages/python/index.js.map +1 -0
- package/dist/languages/python/profile.d.ts +36 -0
- package/dist/languages/python/profile.d.ts.map +1 -0
- package/dist/languages/python/profile.js +282 -0
- package/dist/languages/python/profile.js.map +1 -0
- package/dist/languages/python/signature.d.ts +53 -0
- package/dist/languages/python/signature.d.ts.map +1 -0
- package/dist/languages/python/signature.js +188 -0
- package/dist/languages/python/signature.js.map +1 -0
- package/dist/languages/python/table.d.ts +46 -0
- package/dist/languages/python/table.d.ts.map +1 -0
- package/dist/languages/python/table.js +88 -0
- package/dist/languages/python/table.js.map +1 -0
- package/dist/languages/shared/call-detection.d.ts +17 -0
- package/dist/languages/shared/call-detection.d.ts.map +1 -0
- package/dist/languages/shared/call-detection.js +86 -0
- package/dist/languages/shared/call-detection.js.map +1 -0
- package/dist/languages/shared/chunks.d.ts +28 -0
- package/dist/languages/shared/chunks.d.ts.map +1 -0
- package/dist/languages/shared/chunks.js +181 -0
- package/dist/languages/shared/chunks.js.map +1 -0
- package/dist/languages/shared/create-adapter.d.ts +11 -0
- package/dist/languages/shared/create-adapter.d.ts.map +1 -0
- package/dist/languages/shared/create-adapter.js +90 -0
- package/dist/languages/shared/create-adapter.js.map +1 -0
- package/dist/languages/shared/decisions.d.ts +45 -0
- package/dist/languages/shared/decisions.d.ts.map +1 -0
- package/dist/languages/shared/decisions.js +141 -0
- package/dist/languages/shared/decisions.js.map +1 -0
- package/dist/languages/shared/dialect.d.ts +18 -0
- package/dist/languages/shared/dialect.d.ts.map +1 -0
- package/dist/languages/shared/dialect.js +37 -0
- package/dist/languages/shared/dialect.js.map +1 -0
- package/dist/languages/shared/doc-binding.d.ts +26 -0
- package/dist/languages/shared/doc-binding.d.ts.map +1 -0
- package/dist/languages/shared/doc-binding.js +50 -0
- package/dist/languages/shared/doc-binding.js.map +1 -0
- package/dist/languages/shared/engine.d.ts +49 -0
- package/dist/languages/shared/engine.d.ts.map +1 -0
- package/dist/languages/shared/engine.js +317 -0
- package/dist/languages/shared/engine.js.map +1 -0
- package/dist/languages/shared/extract-symbols.d.ts +5 -0
- package/dist/languages/shared/extract-symbols.d.ts.map +1 -0
- package/dist/languages/shared/extract-symbols.js +31 -0
- package/dist/languages/shared/extract-symbols.js.map +1 -0
- package/dist/languages/shared/fingerprint.d.ts +7 -0
- package/dist/languages/shared/fingerprint.d.ts.map +1 -0
- package/dist/languages/shared/fingerprint.js +95 -0
- package/dist/languages/shared/fingerprint.js.map +1 -0
- package/dist/languages/shared/grammar-loader.d.ts +15 -0
- package/dist/languages/shared/grammar-loader.d.ts.map +1 -0
- package/dist/languages/shared/grammar-loader.js +48 -0
- package/dist/languages/shared/grammar-loader.js.map +1 -0
- package/dist/languages/shared/import-detection.d.ts +24 -0
- package/dist/languages/shared/import-detection.d.ts.map +1 -0
- package/dist/languages/shared/import-detection.js +138 -0
- package/dist/languages/shared/import-detection.js.map +1 -0
- package/dist/languages/shared/js-family-profile.d.ts +5 -0
- package/dist/languages/shared/js-family-profile.d.ts.map +1 -0
- package/dist/languages/shared/js-family-profile.js +179 -0
- package/dist/languages/shared/js-family-profile.js.map +1 -0
- package/dist/languages/shared/node-kinds.d.ts +78 -0
- package/dist/languages/shared/node-kinds.d.ts.map +1 -0
- package/dist/languages/shared/node-kinds.js +141 -0
- package/dist/languages/shared/node-kinds.js.map +1 -0
- package/dist/languages/shared/node-utils.d.ts +31 -0
- package/dist/languages/shared/node-utils.d.ts.map +1 -0
- package/dist/languages/shared/node-utils.js +89 -0
- package/dist/languages/shared/node-utils.js.map +1 -0
- package/dist/languages/shared/parse-diagnostics.d.ts +44 -0
- package/dist/languages/shared/parse-diagnostics.d.ts.map +1 -0
- package/dist/languages/shared/parse-diagnostics.js +64 -0
- package/dist/languages/shared/parse-diagnostics.js.map +1 -0
- package/dist/languages/shared/parse-limits.d.ts +24 -0
- package/dist/languages/shared/parse-limits.d.ts.map +1 -0
- package/dist/languages/shared/parse-limits.js +134 -0
- package/dist/languages/shared/parse-limits.js.map +1 -0
- package/dist/languages/shared/profile.d.ts +149 -0
- package/dist/languages/shared/profile.d.ts.map +1 -0
- package/dist/languages/shared/profile.js +9 -0
- package/dist/languages/shared/profile.js.map +1 -0
- package/dist/languages/shared/scope-model.d.ts +32 -0
- package/dist/languages/shared/scope-model.d.ts.map +1 -0
- package/dist/languages/shared/scope-model.js +2 -0
- package/dist/languages/shared/scope-model.js.map +1 -0
- package/dist/languages/shared/scope-naming.d.ts +20 -0
- package/dist/languages/shared/scope-naming.d.ts.map +1 -0
- package/dist/languages/shared/scope-naming.js +118 -0
- package/dist/languages/shared/scope-naming.js.map +1 -0
- package/dist/languages/shared/scope-signature.d.ts +14 -0
- package/dist/languages/shared/scope-signature.d.ts.map +1 -0
- package/dist/languages/shared/scope-signature.js +82 -0
- package/dist/languages/shared/scope-signature.js.map +1 -0
- package/dist/languages/shared/signature.d.ts +55 -0
- package/dist/languages/shared/signature.d.ts.map +1 -0
- package/dist/languages/shared/signature.js +147 -0
- package/dist/languages/shared/signature.js.map +1 -0
- package/dist/languages/shared/syntax.d.ts +115 -0
- package/dist/languages/shared/syntax.d.ts.map +1 -0
- package/dist/languages/shared/syntax.js +2 -0
- package/dist/languages/shared/syntax.js.map +1 -0
- package/dist/languages/shared/table-profile-decisions.d.ts +54 -0
- package/dist/languages/shared/table-profile-decisions.d.ts.map +1 -0
- package/dist/languages/shared/table-profile-decisions.js +245 -0
- package/dist/languages/shared/table-profile-decisions.js.map +1 -0
- package/dist/languages/shared/table-profile-module-facts.d.ts +29 -0
- package/dist/languages/shared/table-profile-module-facts.d.ts.map +1 -0
- package/dist/languages/shared/table-profile-module-facts.js +139 -0
- package/dist/languages/shared/table-profile-module-facts.js.map +1 -0
- package/dist/languages/shared/table-profile-queries.d.ts +27 -0
- package/dist/languages/shared/table-profile-queries.d.ts.map +1 -0
- package/dist/languages/shared/table-profile-queries.js +70 -0
- package/dist/languages/shared/table-profile-queries.js.map +1 -0
- package/dist/languages/shared/table-profile-scopes.d.ts +46 -0
- package/dist/languages/shared/table-profile-scopes.d.ts.map +1 -0
- package/dist/languages/shared/table-profile-scopes.js +180 -0
- package/dist/languages/shared/table-profile-scopes.js.map +1 -0
- package/dist/languages/shared/table-profile.d.ts +302 -0
- package/dist/languages/shared/table-profile.d.ts.map +1 -0
- package/dist/languages/shared/table-profile.js +125 -0
- package/dist/languages/shared/table-profile.js.map +1 -0
- package/dist/languages/shared/tree-sitter-types.d.ts +44 -0
- package/dist/languages/shared/tree-sitter-types.d.ts.map +1 -0
- package/dist/languages/shared/tree-sitter-types.js +2 -0
- package/dist/languages/shared/tree-sitter-types.js.map +1 -0
- package/dist/languages/structured-text/index.d.ts +48 -0
- package/dist/languages/structured-text/index.d.ts.map +1 -0
- package/dist/languages/structured-text/index.js +120 -0
- package/dist/languages/structured-text/index.js.map +1 -0
- package/dist/languages/tsx/index.d.ts +4 -0
- package/dist/languages/tsx/index.d.ts.map +1 -0
- package/dist/languages/tsx/index.js +7 -0
- package/dist/languages/tsx/index.js.map +1 -0
- package/dist/languages/typescript/index.d.ts +4 -0
- package/dist/languages/typescript/index.d.ts.map +1 -0
- package/dist/languages/typescript/index.js +7 -0
- package/dist/languages/typescript/index.js.map +1 -0
- package/dist/languages/vue/index.d.ts +21 -0
- package/dist/languages/vue/index.d.ts.map +1 -0
- package/dist/languages/vue/index.js +234 -0
- package/dist/languages/vue/index.js.map +1 -0
- package/dist/languages/vue/mapping.d.ts +11 -0
- package/dist/languages/vue/mapping.d.ts.map +1 -0
- package/dist/languages/vue/mapping.js +77 -0
- package/dist/languages/vue/mapping.js.map +1 -0
- package/dist/registry.d.ts +24 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/registry.js +42 -0
- package/dist/registry.js.map +1 -0
- package/dist/text/byte-offset-table.d.ts +27 -0
- package/dist/text/byte-offset-table.d.ts.map +1 -0
- package/dist/text/byte-offset-table.js +39 -0
- package/dist/text/byte-offset-table.js.map +1 -0
- package/package.json +125 -3
- package/src/boundaries/expression.ts +95 -0
- package/src/boundaries/extract.ts +307 -0
- package/src/boundaries/python.ts +181 -0
- package/src/index.ts +51 -0
- package/src/languages/__golden__/javascript-basic.json +32 -0
- package/src/languages/__golden__/tsx-basic.json +19 -0
- package/src/languages/__golden__/typescript-basic.json +37 -0
- package/src/languages/astgrep-backend/GRAMMARS.md +86 -0
- package/src/languages/astgrep-backend/README.md +112 -0
- package/src/languages/astgrep-backend/__golden__/bash-basic.sh +53 -0
- package/src/languages/astgrep-backend/__golden__/c-basic.c +64 -0
- package/src/languages/astgrep-backend/__golden__/cpp-basic.cpp +65 -0
- package/src/languages/astgrep-backend/__golden__/csharp-basic.cs +73 -0
- package/src/languages/astgrep-backend/__golden__/go-basic.go +93 -0
- package/src/languages/astgrep-backend/__golden__/java-basic.java +62 -0
- package/src/languages/astgrep-backend/__golden__/kotlin-basic.kt +58 -0
- package/src/languages/astgrep-backend/__golden__/php-basic.php +72 -0
- package/src/languages/astgrep-backend/__golden__/ruby-basic.rb +62 -0
- package/src/languages/astgrep-backend/__golden__/rust-basic.rs +82 -0
- package/src/languages/astgrep-backend/__golden__/swift-basic.swift +73 -0
- package/src/languages/astgrep-backend/adapter.ts +240 -0
- package/src/languages/astgrep-backend/index.ts +44 -0
- package/src/languages/astgrep-backend/lang/README.md +115 -0
- package/src/languages/astgrep-backend/lang/bash.ts +78 -0
- package/src/languages/astgrep-backend/lang/c.ts +119 -0
- package/src/languages/astgrep-backend/lang/cpp.ts +65 -0
- package/src/languages/astgrep-backend/lang/csharp.ts +125 -0
- package/src/languages/astgrep-backend/lang/index.ts +61 -0
- package/src/languages/astgrep-backend/lang/java.ts +115 -0
- package/src/languages/astgrep-backend/lang/kotlin.ts +136 -0
- package/src/languages/astgrep-backend/lang/php.ts +105 -0
- package/src/languages/astgrep-backend/lang/ruby.ts +105 -0
- package/src/languages/astgrep-backend/lang/swift.ts +118 -0
- package/src/languages/astgrep-backend/languages.ts +238 -0
- package/src/languages/astgrep-backend/napi.ts +261 -0
- package/src/languages/astgrep-backend/syntax.ts +242 -0
- package/src/languages/index.ts +53 -0
- package/src/languages/javascript/index.ts +9 -0
- package/src/languages/jsx/index.ts +9 -0
- package/src/languages/prose/index.ts +270 -0
- package/src/languages/prose/line-bytes.ts +52 -0
- package/src/languages/python/README.md +246 -0
- package/src/languages/python/__golden__/python-basic.json +44 -0
- package/src/languages/python/create-adapter.ts +29 -0
- package/src/languages/python/index.ts +8 -0
- package/src/languages/python/profile.ts +341 -0
- package/src/languages/python/signature.ts +183 -0
- package/src/languages/python/table.ts +135 -0
- package/src/languages/shared/__fixtures__/large-over-32kib.ts.txt +1207 -0
- package/src/languages/shared/call-detection.ts +112 -0
- package/src/languages/shared/chunks.ts +255 -0
- package/src/languages/shared/create-adapter.ts +136 -0
- package/src/languages/shared/decisions.ts +204 -0
- package/src/languages/shared/dialect.ts +55 -0
- package/src/languages/shared/doc-binding.ts +78 -0
- package/src/languages/shared/engine.ts +461 -0
- package/src/languages/shared/extract-symbols.ts +35 -0
- package/src/languages/shared/fingerprint.ts +99 -0
- package/src/languages/shared/grammar-loader.ts +67 -0
- package/src/languages/shared/import-detection.ts +154 -0
- package/src/languages/shared/js-family-profile.ts +246 -0
- package/src/languages/shared/node-kinds.ts +153 -0
- package/src/languages/shared/node-utils.ts +120 -0
- package/src/languages/shared/parse-diagnostics.ts +125 -0
- package/src/languages/shared/parse-limits.ts +147 -0
- package/src/languages/shared/profile.ts +184 -0
- package/src/languages/shared/scope-model.ts +43 -0
- package/src/languages/shared/scope-naming.ts +124 -0
- package/src/languages/shared/scope-signature.ts +108 -0
- package/src/languages/shared/signature.ts +149 -0
- package/src/languages/shared/syntax.ts +116 -0
- package/src/languages/shared/table-profile-decisions.ts +251 -0
- package/src/languages/shared/table-profile-module-facts.ts +177 -0
- package/src/languages/shared/table-profile-queries.ts +85 -0
- package/src/languages/shared/table-profile-scopes.ts +198 -0
- package/src/languages/shared/table-profile.ts +427 -0
- package/src/languages/shared/tree-sitter-types.ts +46 -0
- package/src/languages/structured-text/index.ts +192 -0
- package/src/languages/tsx/index.ts +9 -0
- package/src/languages/typescript/index.ts +9 -0
- package/src/languages/vue/index.ts +312 -0
- package/src/languages/vue/mapping.ts +93 -0
- package/src/registry.ts +61 -0
- package/src/text/byte-offset-table.ts +63 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import type { HttpBoundaryFileFactsV1 } from "@untangle-code/domain";
|
|
2
|
+
|
|
3
|
+
import type { FileTextContext } from "../languages/shared/node-utils.js";
|
|
4
|
+
import { rangeOf } from "../languages/shared/node-utils.js";
|
|
5
|
+
import type { SyntaxNode } from "../languages/shared/syntax.js";
|
|
6
|
+
|
|
7
|
+
import { field, namedChildren, stringValue } from "./expression.js";
|
|
8
|
+
import { scopeOf } from "./extract.js";
|
|
9
|
+
|
|
10
|
+
const METHODS = new Set(["get", "post", "put", "delete", "patch", "head", "options", "trace"]);
|
|
11
|
+
|
|
12
|
+
function keyword(node: SyntaxNode, name: string): SyntaxNode | null {
|
|
13
|
+
return (
|
|
14
|
+
namedChildren(node).find(
|
|
15
|
+
(child) => child.type === "keyword_argument" && field(child, "name")?.text === name,
|
|
16
|
+
) ?? null
|
|
17
|
+
);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function prefix(node: SyntaxNode): string | null {
|
|
21
|
+
const value = keyword(node, "prefix");
|
|
22
|
+
return value === null ? "" : stringValue(field(value, "value"));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function importFacts(
|
|
26
|
+
nodes: readonly SyntaxNode[],
|
|
27
|
+
ctx: FileTextContext,
|
|
28
|
+
facts: HttpBoundaryFileFactsV1,
|
|
29
|
+
): void {
|
|
30
|
+
for (const node of nodes) {
|
|
31
|
+
if (node.type !== "import_from_statement" || scopeOf(node) !== "module") continue;
|
|
32
|
+
const module = field(node, "module_name");
|
|
33
|
+
if (module === null) continue;
|
|
34
|
+
for (const child of namedChildren(node)) {
|
|
35
|
+
if (child.startIndex === module.startIndex) continue;
|
|
36
|
+
const name = child.type === "aliased_import" ? field(child, "name") : child;
|
|
37
|
+
const alias = child.type === "aliased_import" ? field(child, "alias") : child;
|
|
38
|
+
if (
|
|
39
|
+
name === null ||
|
|
40
|
+
alias === null ||
|
|
41
|
+
!["identifier", "dotted_name", "aliased_import"].includes(child.type)
|
|
42
|
+
)
|
|
43
|
+
continue;
|
|
44
|
+
facts.pythonImports.push({
|
|
45
|
+
module: module.text,
|
|
46
|
+
local: alias.text,
|
|
47
|
+
imported: name.text,
|
|
48
|
+
range: rangeOf(child, ctx),
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function expandedIncludeTargets(node: SyntaxNode, target: SyntaxNode): string[] {
|
|
55
|
+
if (target.type !== "identifier") return [target.text];
|
|
56
|
+
let parent = node.parent;
|
|
57
|
+
while (parent !== null) {
|
|
58
|
+
if (parent.type === "for_statement" && field(parent, "left")?.text === target.text) {
|
|
59
|
+
const right = field(parent, "right");
|
|
60
|
+
if (right !== null && ["tuple", "list"].includes(right.type)) {
|
|
61
|
+
const items = namedChildren(right);
|
|
62
|
+
if (items.every((item) => item.type === "attribute" || item.type === "identifier"))
|
|
63
|
+
return items.map((item) => item.text);
|
|
64
|
+
}
|
|
65
|
+
return ["<dynamic-router-iteration>"];
|
|
66
|
+
}
|
|
67
|
+
if (parent.type === "function_definition") break;
|
|
68
|
+
parent = parent.parent;
|
|
69
|
+
}
|
|
70
|
+
return [target.text];
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Only FastAPI-imported constructors, literal prefixes and actual decorated functions produce routes. */
|
|
74
|
+
export function extractPythonHttpFacts(
|
|
75
|
+
nodes: readonly SyntaxNode[],
|
|
76
|
+
ctx: FileTextContext,
|
|
77
|
+
facts: HttpBoundaryFileFactsV1,
|
|
78
|
+
): void {
|
|
79
|
+
importFacts(nodes, ctx, facts);
|
|
80
|
+
const constructorNames = new Map(
|
|
81
|
+
facts.pythonImports
|
|
82
|
+
.filter(
|
|
83
|
+
(item) =>
|
|
84
|
+
item.module === "fastapi" && ["FastAPI", "APIRouter"].includes(item.imported ?? ""),
|
|
85
|
+
)
|
|
86
|
+
.map((item) => [item.local, item.imported]),
|
|
87
|
+
);
|
|
88
|
+
for (const node of nodes) {
|
|
89
|
+
if (node.type === "assignment") {
|
|
90
|
+
const left = field(node, "left"),
|
|
91
|
+
right = field(node, "right");
|
|
92
|
+
if (left?.type !== "identifier" || right?.type !== "call") continue;
|
|
93
|
+
const kind = constructorNames.get(field(right, "function")?.text ?? "");
|
|
94
|
+
const args = field(right, "arguments");
|
|
95
|
+
if (kind === undefined || args === null) continue;
|
|
96
|
+
const constructorName = field(right, "function")?.text;
|
|
97
|
+
let scope = scopeOf(node);
|
|
98
|
+
let parameterShadow = false;
|
|
99
|
+
for (let depth = 0; depth < 30 && scope !== "module"; depth += 1) {
|
|
100
|
+
const fn = facts.functions.find((candidate) => candidate.scope === scope);
|
|
101
|
+
if (fn?.parameters.some((parameter) => parameter.name === constructorName))
|
|
102
|
+
parameterShadow = true;
|
|
103
|
+
scope = fn?.parentScope ?? "module";
|
|
104
|
+
}
|
|
105
|
+
if (
|
|
106
|
+
parameterShadow ||
|
|
107
|
+
facts.pythonImports.filter((item) => item.local === constructorName).length !== 1 ||
|
|
108
|
+
facts.bindings.some((binding) => binding.name === constructorName) ||
|
|
109
|
+
facts.functions.some((fn) => fn.name === constructorName)
|
|
110
|
+
) {
|
|
111
|
+
facts.limitations.push(`FastAPI constructor ${constructorName ?? "<unknown>"} is shadowed`);
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
facts.routers.push({
|
|
115
|
+
name: left.text,
|
|
116
|
+
scope: scopeOf(node),
|
|
117
|
+
kind: kind === "FastAPI" ? "app" : "router",
|
|
118
|
+
prefix: prefix(args),
|
|
119
|
+
range: rangeOf(node, ctx),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
if (node.type === "decorated_definition") {
|
|
123
|
+
const handler = namedChildren(node).find((child) => child.type === "function_definition");
|
|
124
|
+
if (handler === undefined) continue;
|
|
125
|
+
for (const decorator of namedChildren(node).filter((child) => child.type === "decorator")) {
|
|
126
|
+
const call = namedChildren(decorator)[0];
|
|
127
|
+
if (call?.type !== "call") continue;
|
|
128
|
+
const callee = field(call, "function");
|
|
129
|
+
if (callee?.type !== "attribute") continue;
|
|
130
|
+
const method = field(callee, "attribute")?.text;
|
|
131
|
+
const object = field(callee, "object");
|
|
132
|
+
const args = field(call, "arguments");
|
|
133
|
+
if (
|
|
134
|
+
method === undefined ||
|
|
135
|
+
!METHODS.has(method) ||
|
|
136
|
+
object?.type !== "identifier" ||
|
|
137
|
+
args === null
|
|
138
|
+
)
|
|
139
|
+
continue;
|
|
140
|
+
const first = namedChildren(args).find((arg) => arg.type !== "keyword_argument");
|
|
141
|
+
facts.routes.push({
|
|
142
|
+
router: object.text,
|
|
143
|
+
scope: scopeOf(node),
|
|
144
|
+
method: method.toUpperCase(),
|
|
145
|
+
path: stringValue(first ?? null),
|
|
146
|
+
handler: field(handler, "name")?.text ?? "<unnamed>",
|
|
147
|
+
range: rangeOf(decorator, ctx),
|
|
148
|
+
handlerRange: rangeOf(handler, ctx),
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (node.type === "call") {
|
|
153
|
+
const callee = field(node, "function");
|
|
154
|
+
if (callee?.type === "attribute" && field(callee, "attribute")?.text === "mount") {
|
|
155
|
+
facts.limitations.push(
|
|
156
|
+
"ASGI app.mount routing is unsupported; app entrypoints cannot be inferred",
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
if (callee?.type !== "attribute" || field(callee, "attribute")?.text !== "include_router")
|
|
160
|
+
continue;
|
|
161
|
+
const owner = field(callee, "object"),
|
|
162
|
+
args = field(node, "arguments");
|
|
163
|
+
if (owner?.type !== "identifier" || args === null) continue;
|
|
164
|
+
const target = namedChildren(args).find((arg) => arg.type !== "keyword_argument");
|
|
165
|
+
if (target === undefined) continue;
|
|
166
|
+
for (const router of expandedIncludeTargets(node, target))
|
|
167
|
+
facts.includes.push({
|
|
168
|
+
owner: owner.text,
|
|
169
|
+
scope: scopeOf(node),
|
|
170
|
+
router,
|
|
171
|
+
prefix: prefix(args),
|
|
172
|
+
range: rangeOf(node, ctx),
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
// A partial module graph remains visible; no app/include discovery is inferred from path names.
|
|
177
|
+
if (facts.routes.length > 0 && facts.routers.length === 0)
|
|
178
|
+
facts.limitations.push(
|
|
179
|
+
"Route decorators have no proven FastAPI router constructor in this file",
|
|
180
|
+
);
|
|
181
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @untangle-code/parser-registry — the ONLY public export of this package.
|
|
3
|
+
*
|
|
4
|
+
* `createParserRegistry()` returns a `ParserRegistry` (domain `ports.ts`)
|
|
5
|
+
* with the TypeScript/JavaScript/TSX/JSX `LanguageAdapter`s already
|
|
6
|
+
* registered. Registration is cheap: no grammar is loaded until the first
|
|
7
|
+
* `parse()` call for that language (see `registry.ts` / `languages/shared/grammar-loader.ts`).
|
|
8
|
+
*
|
|
9
|
+
* `@untangle-code/core` (WP-23)'s public `createParserRegistry(config)`
|
|
10
|
+
* facade (spec.md §22) wraps this factory.
|
|
11
|
+
*/
|
|
12
|
+
import type { ParserRegistry } from "@untangle-code/domain";
|
|
13
|
+
|
|
14
|
+
import { registerBuiltinLanguages } from "./languages/index.js";
|
|
15
|
+
import { LazyParserRegistry } from "./registry.js";
|
|
16
|
+
export { extractHttpBoundaryFacts } from "./boundaries/extract.js";
|
|
17
|
+
|
|
18
|
+
export function createParserRegistry(): ParserRegistry {
|
|
19
|
+
const registry = new LazyParserRegistry();
|
|
20
|
+
registerBuiltinLanguages(registry);
|
|
21
|
+
return registry;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export { LazyParserRegistry } from "./registry.js";
|
|
25
|
+
export { createTypeScriptAdapter } from "./languages/typescript/index.js";
|
|
26
|
+
export { createJavaScriptAdapter } from "./languages/javascript/index.js";
|
|
27
|
+
export { createTsxAdapter } from "./languages/tsx/index.js";
|
|
28
|
+
export { createJsxAdapter } from "./languages/jsx/index.js";
|
|
29
|
+
export { createVueAdapter } from "./languages/vue/index.js";
|
|
30
|
+
export { createPythonAdapter } from "./languages/python/index.js";
|
|
31
|
+
export {
|
|
32
|
+
createProseAdapter,
|
|
33
|
+
PROSE_LANGUAGE_ID,
|
|
34
|
+
PROSE_SYMBOL_KIND,
|
|
35
|
+
} from "./languages/prose/index.js";
|
|
36
|
+
export {
|
|
37
|
+
createStructuredTextAdapter,
|
|
38
|
+
STRUCTURED_TEXT_LANGUAGE_ID,
|
|
39
|
+
} from "./languages/structured-text/index.js";
|
|
40
|
+
export {
|
|
41
|
+
astGrepLanguageStates,
|
|
42
|
+
astGrepLanguageStatus,
|
|
43
|
+
createAstGrepAdapter,
|
|
44
|
+
createGoAdapter,
|
|
45
|
+
createRustAdapter,
|
|
46
|
+
MAX_ASTGREP_SOURCE_BYTES,
|
|
47
|
+
} from "./languages/astgrep-backend/index.js";
|
|
48
|
+
export type {
|
|
49
|
+
AstGrepLanguageSource,
|
|
50
|
+
AstGrepLanguageStatus,
|
|
51
|
+
} from "./languages/astgrep-backend/index.js";
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"languageId": "javascript",
|
|
3
|
+
"path": "basic.js",
|
|
4
|
+
"source": "const EventEmitterLike = require(\"./emitter\");\n\nexport class Logger {\n #level = \"info\";\n\n log(message) {\n if (this.#level === \"info\" || this.#level === \"debug\") {\n console.log(message);\n }\n }\n}\n\nexport function createLogger() {\n return new Logger();\n}\n",
|
|
5
|
+
"expectedSymbols": [
|
|
6
|
+
{ "name": "Logger", "kind": "class", "qualifiedName": "Logger", "exportedNames": ["Logger"] },
|
|
7
|
+
{
|
|
8
|
+
"name": "#level",
|
|
9
|
+
"kind": "property",
|
|
10
|
+
"qualifiedName": "Logger.#level",
|
|
11
|
+
"exportedNames": [],
|
|
12
|
+
"visibility": "private"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"name": "log",
|
|
16
|
+
"kind": "method",
|
|
17
|
+
"qualifiedName": "Logger.log",
|
|
18
|
+
"normalizedSignature": "log(message)",
|
|
19
|
+
"exportedNames": []
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"name": "createLogger",
|
|
23
|
+
"kind": "function",
|
|
24
|
+
"qualifiedName": "createLogger",
|
|
25
|
+
"normalizedSignature": "createLogger()",
|
|
26
|
+
"exportedNames": ["createLogger"]
|
|
27
|
+
}
|
|
28
|
+
],
|
|
29
|
+
"expectedImports": [{ "specifier": "./emitter", "importKind": "import" }],
|
|
30
|
+
"expectedCalls": [{ "calleeName": "log", "qualifier": "console" }, { "calleeName": "Logger" }],
|
|
31
|
+
"expectedDecisionKinds": ["if", "logical_or"]
|
|
32
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"languageId": "tsx",
|
|
3
|
+
"path": "basic.tsx",
|
|
4
|
+
"source": "import { useState } from \"react\";\n\nexport function Counter({ start }: { start: number }) {\n const [count, setCount] = useState(start);\n return (\n <button onClick={() => setCount(count + 1)}>\n {count > 0 ? \"positive\" : \"zero or negative\"}\n </button>\n );\n}\n",
|
|
5
|
+
"expectedSymbols": [
|
|
6
|
+
{
|
|
7
|
+
"name": "Counter",
|
|
8
|
+
"kind": "function",
|
|
9
|
+
"qualifiedName": "Counter",
|
|
10
|
+
"normalizedSignature": "Counter({ start }: { start: number })",
|
|
11
|
+
"exportedNames": ["Counter"]
|
|
12
|
+
}
|
|
13
|
+
],
|
|
14
|
+
"expectedImports": [
|
|
15
|
+
{ "specifier": "react", "importKind": "import", "importedNames": ["useState"] }
|
|
16
|
+
],
|
|
17
|
+
"expectedCalls": [{ "calleeName": "useState" }, { "calleeName": "setCount" }],
|
|
18
|
+
"expectedDecisionKinds": ["ternary"]
|
|
19
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"languageId": "typescript",
|
|
3
|
+
"path": "basic.ts",
|
|
4
|
+
"source": "import { readFile } from \"node:fs\";\nexport { helper } from \"./helper\";\n\n/** Adds two numbers together. */\nexport function add(a: number, b: number = 1): number {\n if (a > 0 && b > 0) {\n return a + b;\n } else if (a < 0) {\n return -1;\n }\n return 0;\n}\n\nexport interface Point {\n x: number;\n y: number;\n}\n\nexport class Box {\n private width: number = 0;\n\n resize(value: number): void {\n this.width = value;\n }\n}\n",
|
|
5
|
+
"expectedSymbols": [
|
|
6
|
+
{
|
|
7
|
+
"name": "add",
|
|
8
|
+
"kind": "function",
|
|
9
|
+
"qualifiedName": "add",
|
|
10
|
+
"normalizedSignature": "add(a: number, b: number): number",
|
|
11
|
+
"exportedNames": ["add"],
|
|
12
|
+
"hasDocRange": true
|
|
13
|
+
},
|
|
14
|
+
{ "name": "Point", "kind": "interface", "qualifiedName": "Point", "exportedNames": ["Point"] },
|
|
15
|
+
{ "name": "Box", "kind": "class", "qualifiedName": "Box", "exportedNames": ["Box"] },
|
|
16
|
+
{
|
|
17
|
+
"name": "width",
|
|
18
|
+
"kind": "property",
|
|
19
|
+
"qualifiedName": "Box.width",
|
|
20
|
+
"normalizedSignature": "private width: number",
|
|
21
|
+
"exportedNames": []
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"name": "resize",
|
|
25
|
+
"kind": "method",
|
|
26
|
+
"qualifiedName": "Box.resize",
|
|
27
|
+
"normalizedSignature": "resize(value: number): void",
|
|
28
|
+
"exportedNames": []
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
"expectedImports": [
|
|
32
|
+
{ "specifier": "node:fs", "importKind": "import", "importedNames": ["readFile"] },
|
|
33
|
+
{ "specifier": "./helper", "importKind": "re_export", "importedNames": ["helper"] }
|
|
34
|
+
],
|
|
35
|
+
"expectedCallCount": 0,
|
|
36
|
+
"expectedDecisionKinds": ["if", "logical_and", "else_if"]
|
|
37
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
<a id="die-eingebetteten-grammatiken--herkunft-und-lizenz"></a>
|
|
2
|
+
|
|
3
|
+
# Embedded grammars — provenance and licensing
|
|
4
|
+
|
|
5
|
+
**Why this file is maintained manually** while `THIRD_PARTY_LICENSES.md` is generated:
|
|
6
|
+
`pnpm third-party-licenses` reads the **production dependency tree**. Grammars are absent there:
|
|
7
|
+
they are `devDependencies` **inside** `@ast-grep/lang-*` packages, used to generate `src/parser.c`
|
|
8
|
+
and `prebuilds/*/parser.so`. Neither the generator nor `scripts/check-licenses.mjs` can see them.
|
|
9
|
+
|
|
10
|
+
**The language packages themselves do not identify them.** An `@ast-grep/lang-*` LICENSE lists
|
|
11
|
+
only *ast-grep* as copyright holder despite shipping another tree-sitter grammar. Relying on it
|
|
12
|
+
would omit the actual authors. This mapping was checked in each installed language package's
|
|
13
|
+
`devDependencies` block in `node_modules`, not guessed.
|
|
14
|
+
|
|
15
|
+
<a id="sprachpakete-optionale-peer-abhängigkeiten-von-untangle-codeparser-registry"></a>
|
|
16
|
+
|
|
17
|
+
## Language packages (optional peers of `@untangle-code/parser-registry`)
|
|
18
|
+
|
|
19
|
+
| Language package | Version | `package.json` license | `LICENSE` file license |
|
|
20
|
+
|---|---|---|---|
|
|
21
|
+
| `@ast-grep/lang-go` | 0.0.6 | ISC | MIT (ast-grep) |
|
|
22
|
+
| `@ast-grep/lang-rust` | 0.0.7 | ISC | MIT (ast-grep) |
|
|
23
|
+
| `@ast-grep/lang-java` | 0.0.7 | ISC | MIT (ast-grep) |
|
|
24
|
+
| `@ast-grep/lang-csharp` | 0.0.6 | ISC | MIT (ast-grep) |
|
|
25
|
+
| `@ast-grep/lang-ruby` | 0.0.7 | ISC | MIT (ast-grep) |
|
|
26
|
+
| `@ast-grep/lang-php` | 0.0.7 | ISC | MIT (ast-grep) |
|
|
27
|
+
| `@ast-grep/lang-kotlin` | 0.0.7 | ISC | MIT (ast-grep) |
|
|
28
|
+
| `@ast-grep/lang-c` | 0.0.6 | ISC | MIT (ast-grep) |
|
|
29
|
+
| `@ast-grep/lang-cpp` | 0.0.6 | ISC | MIT (ast-grep) |
|
|
30
|
+
| `@ast-grep/lang-swift` | 0.0.8 | ISC | MIT (ast-grep) |
|
|
31
|
+
| `@ast-grep/lang-bash` | 0.0.8 | ISC | MIT (ast-grep) |
|
|
32
|
+
|
|
33
|
+
ISC and MIT are both allowed by `scripts/check-licenses.mjs` and compatible with AGPL-3.0-only.
|
|
34
|
+
The disagreement between `package.json` and LICENSE is a package characteristic, recorded rather
|
|
35
|
+
than smoothed over.
|
|
36
|
+
|
|
37
|
+
<a id="die-tatsächlich-eingebetteten-grammatiken"></a>
|
|
38
|
+
|
|
39
|
+
## Grammars actually embedded
|
|
40
|
+
|
|
41
|
+
From each language package's `devDependencies`. Every grammar is a separate work with its own
|
|
42
|
+
authors; its license continues to apply to shipped `parser.c` and derived `parser.so` files.
|
|
43
|
+
|
|
44
|
+
| Language package | Embedded grammar | Version | License |
|
|
45
|
+
|---|---|---|---|
|
|
46
|
+
| `@ast-grep/lang-go` | `tree-sitter-go` | 0.25.0 | MIT |
|
|
47
|
+
| `@ast-grep/lang-rust` | `tree-sitter-rust` | — | MIT |
|
|
48
|
+
| `@ast-grep/lang-java` | `tree-sitter-java` | 0.23.5 | MIT |
|
|
49
|
+
| `@ast-grep/lang-csharp` | `tree-sitter-c-sharp` | 0.23.1 | MIT |
|
|
50
|
+
| `@ast-grep/lang-ruby` | `tree-sitter-ruby` | 0.23.1 | MIT |
|
|
51
|
+
| `@ast-grep/lang-php` | `tree-sitter-php` | 0.24.2 | MIT |
|
|
52
|
+
| `@ast-grep/lang-kotlin` | `tree-sitter-kotlin` | 0.3.8 | MIT |
|
|
53
|
+
| `@ast-grep/lang-c` | `tree-sitter-c` | 0.24.1 | MIT |
|
|
54
|
+
| `@ast-grep/lang-cpp` | `tree-sitter-cpp` | 0.23.4 | MIT |
|
|
55
|
+
| `@ast-grep/lang-swift` | `tree-sitter-swift` | 0.7.1 | MIT |
|
|
56
|
+
| `@ast-grep/lang-bash` | `tree-sitter-bash` | 0.25.1 | MIT |
|
|
57
|
+
|
|
58
|
+
All eleven use **MIT**, compatible with AGPL. Checked on 2026-09-05 against the registry
|
|
59
|
+
(`npm view <package> license`) and installed language packages' `devDependencies`.
|
|
60
|
+
|
|
61
|
+
<a id="warum-die-sprachpakete-optionale-peer-abhängigkeiten-sind"></a>
|
|
62
|
+
|
|
63
|
+
## Why language packages are OPTIONAL peers
|
|
64
|
+
|
|
65
|
+
Together the eleven packages occupy **294 MB** (measured through
|
|
66
|
+
`du -sh node_modules/.pnpm/@ast-grep+lang-*`; Swift alone is 69 MB, C# 62 MB, Kotlin 42 MB).
|
|
67
|
+
Every package ships C source AND prebuilds for five platforms.
|
|
68
|
+
|
|
69
|
+
Normal `dependencies` would force every Untangle installation to download all eleven, even for
|
|
70
|
+
TypeScript-only use. As **optional peers**, users install only what they need. An absent grammar
|
|
71
|
+
produces `grammar_unavailable` with a named reason on first parse; callers turn that into a
|
|
72
|
+
skip-ledger entry (E4). No adapter pretends to have parsed.
|
|
73
|
+
|
|
74
|
+
<a id="plattformen-ohne-prebuild"></a>
|
|
75
|
+
|
|
76
|
+
## Platforms without prebuilds
|
|
77
|
+
|
|
78
|
+
Core `@ast-grep/napi` covers nine targets. **Language packages** cover only five: macOS x64,
|
|
79
|
+
macOS arm64, Linux x64 glibc, Linux arm64 glibc and Windows x64.
|
|
80
|
+
|
|
81
|
+
**Missing: Linux musl (Alpine), Windows arm64, Windows ia32, FreeBSD.** There, installing a
|
|
82
|
+
language package **silently** fails: `postinstall` catches the build error and exits 0, so npm
|
|
83
|
+
and pnpm report success. It becomes visible only when `napi.ts` reads `libraryPath` on first parse.
|
|
84
|
+
|
|
85
|
+
This claim is **derived from package contents**—the existing `prebuilds/` directories—not executed
|
|
86
|
+
on those platforms. Every measurement here ran on darwin/arm64.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
<a id="ast-grep-backend-wp-220"></a>
|
|
2
|
+
|
|
3
|
+
# ast-grep backend (WP-220)
|
|
4
|
+
|
|
5
|
+
This directory is the parser registry's SECOND backend. It uses the same `LanguageAdapter`
|
|
6
|
+
contract as tree-sitter: `chunker`, `complexity`, `file-graph` and `symbol-graph` cannot tell which
|
|
7
|
+
backend ran.
|
|
8
|
+
|
|
9
|
+
<a id="die-regel-welche-sprache-laeuft-wo"></a>
|
|
10
|
+
|
|
11
|
+
## Rule: which language uses which backend
|
|
12
|
+
|
|
13
|
+
| Language | Backend | Reason |
|
|
14
|
+
|---|---|---|
|
|
15
|
+
| TypeScript, JavaScript, TSX, JSX | **tree-sitter** | Measured 3.74× faster traversal, lower memory use, complete `nodeIsMissing`, unchanged golden tests |
|
|
16
|
+
| Python | Custom parser | Unchanged; replacement is prepared, not performed (see Open work) |
|
|
17
|
+
| Markdown/prose, structured data | Custom adapters | `text` support, no syntax tree needed |
|
|
18
|
+
| **Go, Rust** | **ast-grep** | Ready-made MIT-licensed grammars; these languages would not exist here without this backend |
|
|
19
|
+
|
|
20
|
+
The split follows measurement rather than preference. ast-grep has **no cursor API**: descent
|
|
21
|
+
uses `children()`, creating a fresh JS array of node wrappers every time. Repository-wide
|
|
22
|
+
measurement (721 files, 1,143,212 nodes, both versions in one process):
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
tree-sitter Cursor : parse 256 ms walk 671 ms => 1,703,642 nodes/s
|
|
26
|
+
ast-grep children(): parse 347 ms walk 2511 ms => 455,329 nodes/s
|
|
27
|
+
Factor 3.74; overhead 1.61 us per node
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
This matches the `.children` approach removed by WP-95. Switching the JS/TS family would regress
|
|
31
|
+
the exact area repaired in wave 6, while each current dialect costs only nine lines. For Go and
|
|
32
|
+
Rust the overhead is not a regression: those languages would not otherwise exist here.
|
|
33
|
+
|
|
34
|
+
<a id="was-dieses-backend-anders-macht--und-wo-es-steht"></a>
|
|
35
|
+
|
|
36
|
+
## Backend differences and their locations
|
|
37
|
+
|
|
38
|
+
| Aspect | tree-sitter | ast-grep | Location |
|
|
39
|
+
|---|---|---|---|
|
|
40
|
+
| Traversal | Native cursor API | Iterative stack-based cursor | `syntax.ts` |
|
|
41
|
+
| 32-KiB buffer limit | Exists; needs `bufferSize` (WP-215) | Absent, tested to 4 MB without an option | `shared/parse-limits.ts`, `adapter.ts` |
|
|
42
|
+
| Size limit | 8 MiB | **5 MiB**, traversal uses more memory | `adapter.ts` |
|
|
43
|
+
| `missingNodeCount` | Measured through `nodeIsMissing` | **Heuristic**, zero-width leaf | `syntax.ts`, `shared/parse-diagnostics.ts` |
|
|
44
|
+
| Node field name | `cursor.currentFieldName` | Lookup through parent fields | `syntax.ts` |
|
|
45
|
+
| `parser_version` | `untangle-parser-v1` | `untangle-parser-astgrep-v1` | Domain `versions.ts` |
|
|
46
|
+
|
|
47
|
+
EVERY diagnostic derived from heuristic `missingNodeCount` describes its source: heuristically
|
|
48
|
+
detected zero-width nodes; ast-grep has no nodeIsMissing. Anyone reading the count sees its origin,
|
|
49
|
+
the difference between an estimate and a silent assertion (E4).
|
|
50
|
+
|
|
51
|
+
<a id="der-vereinheitlichte-walk-bedient-beide-wege"></a>
|
|
52
|
+
|
|
53
|
+
## One traversal serves BOTH backends
|
|
54
|
+
|
|
55
|
+
`languages/shared/` has been backend-neutral since WP-220:
|
|
56
|
+
|
|
57
|
+
- `syntax.ts`: `SyntaxNode`/`SyntaxCursor` tree contracts. Native tree-sitter types satisfy them
|
|
58
|
+
STRUCTURALLY, without wrappers; the faster backend pays no abstraction cost, pinned in
|
|
59
|
+
`tree-sitter-types.ts`.
|
|
60
|
+
- `profile.ts`: language profile describing node-kind meaning.
|
|
61
|
+
- `js-family-profile.ts`: handwritten TS/JS/TSX/JSX profile, with unchanged behavior (golden tests).
|
|
62
|
+
- `table-profile.ts`: table-driven profiles; a new language costs a table rather than another engine.
|
|
63
|
+
|
|
64
|
+
Wave 6's comment-node defect (`cursor-walk.test.ts`) stays fixed: traversal descends from the ROOT
|
|
65
|
+
and both fingerprints omit comments. The defect does not arise with ast-grep.
|
|
66
|
+
|
|
67
|
+
<a id="eine-sprache-hinzufuegen"></a>
|
|
68
|
+
|
|
69
|
+
## Adding a language
|
|
70
|
+
|
|
71
|
+
1. Add its `@ast-grep/lang-<x>` package INDIVIDUALLY to `package.json`, never a bundle: 22 language
|
|
72
|
+
packages occupy 450 MB together.
|
|
73
|
+
2. Write its table in `languages.ts`, following `GO_TABLE`/`RUST_TABLE`.
|
|
74
|
+
3. Register the adapter in `languages/index.ts`.
|
|
75
|
+
4. Extend `adapter.test.ts` with real source, not a ten-line snippet.
|
|
76
|
+
|
|
77
|
+
<a id="bekannte-grenzen-e4"></a>
|
|
78
|
+
|
|
79
|
+
## Known limits (E4)
|
|
80
|
+
|
|
81
|
+
- **No prebuild on Alpine (Linux musl), Windows ARM64 or Windows ia32.** Language packages ship
|
|
82
|
+
prebuilds only for macOS x64/arm64, Linux x64/arm64 glibc and Windows x64. Missing-prebuild build
|
|
83
|
+
failures are caught and merely logged by `postinstall`, which exits 0, so installation REPORTS
|
|
84
|
+
SUCCESS. `napi.ts` therefore checks each language at registration and reports
|
|
85
|
+
`grammar_unavailable`, producing reasoned skip-ledger entries rather than silent zeros.
|
|
86
|
+
- **One registration only.** `registerDynamicLanguage` runs once; a second call silently
|
|
87
|
+
registers nothing. Every enrolled language is therefore registered TOGETHER on first parse.
|
|
88
|
+
Subsequent enrollment throws rather than doing nothing.
|
|
89
|
+
- **`SgNode#id()` is address-derived** and differs across processes. It serves only field-name
|
|
90
|
+
comparison within a traversal and appears in no result data. `adapter.test.ts` checks
|
|
91
|
+
determinism across two runs.
|
|
92
|
+
- **`Pos.index` is a UTF-16 offset**, despite ast-grep's "byte offset" type documentation.
|
|
93
|
+
All byte ranges use `ByteOffsetTable`.
|
|
94
|
+
- **Table-driven profiles are weaker than handwritten profiles.** Import bindings, call
|
|
95
|
+
qualifiers, signature normalization and Rust `fn` inside `impl` being `function` rather than
|
|
96
|
+
`method` are documented in `shared/table-profile.ts` and the tables themselves.
|
|
97
|
+
- **`LanguageAdapter.parse` is synchronous.** Exploration recommended yielding the event loop per
|
|
98
|
+
file for napi finalizers, but that conflicts with domain `analysis.ts` and was NOT done. A lower
|
|
99
|
+
size limit is used instead; callers parsing many large files in one turn should account for this.
|
|
100
|
+
|
|
101
|
+
<a id="offen-bewusst-nicht-getan"></a>
|
|
102
|
+
|
|
103
|
+
## Open work (deliberately not performed)
|
|
104
|
+
|
|
105
|
+
- **Python retains its custom parser.** Exploration recommends replacing it first, explicitly
|
|
106
|
+
conditional on running the 1,432 lines of Python tests—including WP-76's 20,000-clause `elif`—
|
|
107
|
+
against ast-grep BEFOREHAND. That did not happen, so nothing was deleted. Machinery is ready;
|
|
108
|
+
replacement is a separate work item with its own evidence.
|
|
109
|
+
- **`doctor` does not yet report ast-grep availability.** `napi.ts` exposes per-language
|
|
110
|
+
`astGrepLanguageStates`; the command lacks its display.
|
|
111
|
+
- **THIRD_PARTY_LICENSES.** Language packages ship external tree-sitter grammars while naming only
|
|
112
|
+
ast-grep in LICENSE. The license of EACH grammar needs to be added.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Deploys the app.
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
|
|
5
|
+
MAX_RETRIES=3
|
|
6
|
+
readonly TARGET="${1:-staging}"
|
|
7
|
+
|
|
8
|
+
source ./lib/common.sh
|
|
9
|
+
. ./lib/log.sh
|
|
10
|
+
|
|
11
|
+
log_info() {
|
|
12
|
+
printf '%s\n' "$*" >&2
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function retry() {
|
|
16
|
+
local attempt=0
|
|
17
|
+
while [ "$attempt" -lt "$MAX_RETRIES" ]; do
|
|
18
|
+
if "$@"; then
|
|
19
|
+
return 0
|
|
20
|
+
elif [ "$attempt" -gt 1 ]; then
|
|
21
|
+
log_info "retrying"
|
|
22
|
+
fi
|
|
23
|
+
attempt=$((attempt + 1))
|
|
24
|
+
done
|
|
25
|
+
return 1
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
deploy() {
|
|
29
|
+
case "$TARGET" in
|
|
30
|
+
staging)
|
|
31
|
+
log_info "staging"
|
|
32
|
+
;;
|
|
33
|
+
prod|production)
|
|
34
|
+
log_info "prod"
|
|
35
|
+
;;
|
|
36
|
+
*)
|
|
37
|
+
return 1
|
|
38
|
+
;;
|
|
39
|
+
esac
|
|
40
|
+
|
|
41
|
+
for f in ./dist/*; do
|
|
42
|
+
[ -f "$f" ] || continue
|
|
43
|
+
if [ -x "$f" ] && [ -s "$f" ]; then
|
|
44
|
+
log_info "$f"
|
|
45
|
+
fi
|
|
46
|
+
done
|
|
47
|
+
|
|
48
|
+
until retry true; do
|
|
49
|
+
break
|
|
50
|
+
done
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
deploy
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/* A ring buffer. */
|
|
2
|
+
#include <stdlib.h>
|
|
3
|
+
#include <string.h>
|
|
4
|
+
#include "ring.h"
|
|
5
|
+
|
|
6
|
+
#define RING_MAX 64
|
|
7
|
+
|
|
8
|
+
typedef struct ring {
|
|
9
|
+
char *data;
|
|
10
|
+
size_t head;
|
|
11
|
+
size_t len;
|
|
12
|
+
} ring_t;
|
|
13
|
+
|
|
14
|
+
enum ring_state { RING_OK, RING_FULL };
|
|
15
|
+
|
|
16
|
+
static size_t clamp(size_t v, size_t hi) {
|
|
17
|
+
return v > hi ? hi : v;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/* Creates a ring. */
|
|
21
|
+
ring_t *ring_new(size_t cap) {
|
|
22
|
+
if (cap == 0 || cap > RING_MAX) {
|
|
23
|
+
return NULL;
|
|
24
|
+
}
|
|
25
|
+
ring_t *r = malloc(sizeof(ring_t));
|
|
26
|
+
if (!r) return NULL;
|
|
27
|
+
r->data = malloc(cap);
|
|
28
|
+
r->head = 0;
|
|
29
|
+
r->len = 0;
|
|
30
|
+
return r;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
int ring_push(ring_t *r, const char *src, size_t n) {
|
|
34
|
+
if (r == NULL || src == NULL) {
|
|
35
|
+
return -1;
|
|
36
|
+
} else if (n == 0) {
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
for (size_t i = 0; i < n; i++) {
|
|
40
|
+
while (r->len >= RING_MAX) {
|
|
41
|
+
r->len--;
|
|
42
|
+
}
|
|
43
|
+
switch (src[i]) {
|
|
44
|
+
case '\n':
|
|
45
|
+
r->head = 0;
|
|
46
|
+
break;
|
|
47
|
+
case '\0':
|
|
48
|
+
break;
|
|
49
|
+
default:
|
|
50
|
+
r->data[r->head++] = src[i];
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
do {
|
|
54
|
+
r->len = clamp(r->len, RING_MAX);
|
|
55
|
+
} while (0);
|
|
56
|
+
return (int)n;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
void ring_free(ring_t *r) {
|
|
60
|
+
if (r) {
|
|
61
|
+
free(r->data);
|
|
62
|
+
free(r);
|
|
63
|
+
}
|
|
64
|
+
}
|