@mnemonik/shared 7.144.2 → 7.145.1

package/scripts/vendor-grammars.ts

@@ -11,13 +11,13 @@
  * so there is no peer graph to satisfy, `npm ci` stays fast for every developer
  * and every CI run, and nobody downloads ~283 MB of grammars to build the repo.
  *
- * What ships to a user: the extracted `.wasm` and `.tags.scm` files, committed
+ * What ships to a user: deterministic `.wasm.gz` and `.tags.scm` files, committed
  * here and published inside @mnemonik/shared. A user's install therefore pulls
  * no grammar package, runs no postinstall, needs no compiler, and fetches
  * nothing at runtime - which is the whole reason this plan chose WebAssembly
  * over the native bindings.
  *
- * Run: npx tsx packages/shared/scripts/vendor-grammars.ts [--check | --only=<id>]
+ * Run: npx tsx packages/shared/scripts/vendor-grammars.ts [--check | --only=<id>[,<id>...]]
  *   JSX repair: run --only=javascript and --only=tsx; other assets stay untouched.
  *   --check  verify every manifest entry is present and non-empty; write nothing.
  *            This is what CI should run to catch a grammar that silently stopped
@@ -40,6 +40,7 @@ import {
 import { tmpdir } from 'node:os';
 import { basename, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { gzipSync, gunzipSync } from 'node:zlib';
 import { Language, Parser, Query } from 'web-tree-sitter';
 import { AST_LANGUAGE_IDS, QUERY_CHAIN, grammarArtifactBasename } from '../src/ast/grammars.js';
 
@@ -64,6 +65,10 @@ interface GrammarSpec {
    * maintain when the upstream query is absent or captures unsuitable ranges.
    */
   tags: 'upstream' | 'ours' | 'inherited';
+  /** Compile committed source, regenerating only obsolete parser ABIs. */
+  build?: 'source' | 'regenerate';
+  /** Copy the source license alongside new grammar artifacts. */
+  licenseFile?: string;
 }
 
 /**
@@ -72,23 +77,103 @@ interface GrammarSpec {
  * and losing a language is exactly the silent degradation this plan exists to
  * remove.
  *
- * Absent on purpose:
- *   julia, haskell - large grammars that ALSO ship no tag query, so each
- *     costs both its size (6.1/3.7 MB) and a hand-written query. Dropped to
- *     bound the payload; still indexed via the
- *     allowlist, still chunked heuristically.
- *   dart - its bundled .wasm (package v1.0.0, 2023) fails Language.load under
- *     web-tree-sitter 0.26.11 with a dylink-metadata error: the artifact
- *     predates the current wasm ABI. Presence is not loadability, which is why
- *     --check verifies both. Flutter matters, so this is worth revisiting by
- *     compiling from source with tree-sitter-cli.
- *   kotlin, perl, zig - bundle no .wasm, so they need a build step
- *     with tree-sitter-cli. Deferred until the grammars above are proven; they
- *     keep the heuristic chunker meanwhile.
- *   markdown - keeps chunkMarkdown. Headers already are a document's semantic
- *     unit; an AST would not improve on them.
+ * Markdown retains its document-header chunker. Svelte component parsing is
+ * owned by its adapter; it does not need another tree-sitter WASM.
  */
 const GRAMMARS: GrammarSpec[] = [
+  // fwcd MIT 0.3.8, e1a2d5ad1f61f5740677183cd4125bb071cd2f30, ABI 14.
+  {
+    id: 'kotlin',
+    pkg: 'tree-sitter-kotlin@0.3.8',
+    tags: 'ours',
+    build: 'source',
+    licenseFile: 'LICENSE',
+  },
+  // UserNobody14 ISC 1.0.0, 53485a8f301254e19c518aa20c80f1bcf7cf5c62.
+  // Rebuild ABI-14 source: its published WASM has obsolete dynamic-link metadata.
+  {
+    id: 'dart',
+    pkg: 'tree-sitter-dart@1.0.0',
+    tags: 'ours',
+    build: 'source',
+    licenseFile: 'LICENSE',
+  },
+  // WhatsApp Apache-2.0, committed ABI 15. The unscoped npm name is a security holder.
+  {
+    id: 'erlang',
+    pkg: 'https://codeload.github.com/WhatsApp/tree-sitter-erlang/tar.gz/62c82e67b73e13f3ef06e4c8c07ff035a8235d30',
+    tags: 'ours',
+    build: 'source',
+    licenseFile: 'LICENSE',
+  },
+  // MIT 0.23.1, c30d812bc90827f1a54106a25bc9a6307f5cdcec, packaged ABI 14.
+  { id: 'haskell', pkg: 'tree-sitter-haskell@0.23.1', tags: 'ours', licenseFile: 'LICENSE' },
+  // MIT 0.24.2, 0cc270ff90ca09c29d0f2f9dec69ddfef55a3eff, packaged ABI 14.
+  {
+    id: 'ocaml',
+    pkg: 'tree-sitter-ocaml@0.24.2',
+    wasmFile: 'tree-sitter-ocaml.wasm',
+    tags: 'ours',
+    licenseFile: 'LICENSE',
+  },
+  {
+    id: 'ocaml_interface',
+    pkg: 'tree-sitter-ocaml@0.24.2',
+    wasmFile: 'tree-sitter-ocaml_interface.wasm',
+    tags: 'ours',
+    licenseFile: 'LICENSE',
+  },
+  // oakmac MIT 0.4.0, 290a67a3368be3847b1dd15327bd3a63f2f671ad; regenerate ABI 9.
+  {
+    id: 'clojure',
+    pkg: 'tree-sitter-clojure@0.4.0',
+    tags: 'ours',
+    build: 'regenerate',
+    licenseFile: 'LICENSE.md',
+  },
+  // MIT 0.23.1, a8e1262997d5a45520a06cbe1b86c0737d507054, packaged ABI 14.
+  { id: 'julia', pkg: 'tree-sitter-julia@0.23.1', tags: 'ours', licenseFile: 'LICENSE' },
+  // MIT 2.0.0, 50904961d6a87c5191e611276aa2ecb9d66ca4ff, committed ABI 15.
+  {
+    id: 'perl',
+    pkg: 'tree-sitter-perl@2.0.0',
+    tags: 'ours',
+    build: 'source',
+    licenseFile: 'LICENSE',
+  },
+  // r-lib MIT 1.3.0, committed ABI 14. The unscoped npm name is a security holder.
+  {
+    id: 'r',
+    pkg: 'https://codeload.github.com/r-lib/tree-sitter-r/tar.gz/58a22794466c0fc15b0d3b40531db751593721e8',
+    tags: 'ours',
+    build: 'source',
+    licenseFile: 'LICENSE',
+  },
+  // GrayJack BSD-3-Clause 0.2.0, fb8e54448f19f5d78d7cdc664b35bffa8ec48876; regenerate ABI 10.
+  {
+    id: 'zig',
+    pkg: 'tree-sitter-zig@0.2.0',
+    tags: 'ours',
+    build: 'regenerate',
+    licenseFile: 'LICENSE',
+  },
+  // treywood MIT 0.1.0, committed ABI 14; verified upstream source and license.
+  {
+    id: 'protobuf',
+    pkg: 'https://codeload.github.com/treywood/tree-sitter-proto/tar.gz/e9f6b43f6844bd2189b50a422d4e2094313f6aa3',
+    wasmFile: 'tree-sitter-proto.wasm',
+    tags: 'ours',
+    build: 'source',
+    licenseFile: 'LICENSE',
+  },
+  // bkegley MIT 0.0.1, committed ABI 13; avoid the unrelated ISC npm package.
+  {
+    id: 'graphql',
+    pkg: 'https://codeload.github.com/bkegley/tree-sitter-graphql/tar.gz/5e66e961eee421786bdda8495ed1db045e06b5fe',
+    tags: 'ours',
+    build: 'regenerate',
+    licenseFile: 'LICENSE',
+  },
   {
     // Uy Ha 0.7.5, MIT; build the committed ABI-14 source without native dependencies.
     id: 'cmake',
@@ -123,7 +208,7 @@ const GRAMMARS: GrammarSpec[] = [
   // MIT 0.23.4, gitHead f41e1a044c8a84ea9fa8577fdd2eab92ec96de02.
   { id: 'cpp', pkg: 'tree-sitter-cpp@0.23.4', tags: 'upstream' },
   {
-    // MIT Objective-C++ 0.1.0; committed ABI-15 parser, not the pure Objective-C npm grammar.
+    // MIT Objective-C++ 0.1.0; pinned source, not the pure Objective-C npm grammar.
     id: 'objcpp',
     pkg: 'https://tangled.org/dunkirk.sh/dots',
     wasmFile: 'tree-sitter-objcpp.wasm',
@@ -215,7 +300,7 @@ async function checkOnly(): Promise<Problem[]> {
   await Parser.init();
 
   for (const spec of GRAMMARS) {
-    const wasm = join(WASM_DIR, `${spec.id}.wasm`);
+    const wasm = join(WASM_DIR, `${spec.id}.wasm.gz`);
     if (!existsSync(wasm) || statSync(wasm).size === 0) {
       problems.push({ id: spec.id, detail: `missing or empty ${spec.id}.wasm` });
       continue;
@@ -223,7 +308,7 @@ async function checkOnly(): Promise<Problem[]> {
 
     let language: Language;
     try {
-      language = await Language.load(wasm);
+      language = await Language.load(gunzipSync(readFileSync(wasm)));
     } catch (err) {
       problems.push({
         id: spec.id,
@@ -277,9 +362,9 @@ async function checkOnly(): Promise<Problem[]> {
 }
 
 /**
- * 0.23.2's published WASMs and generated C share these two grammar defects.
- * Keep `unique` contextual, and let GLR distinguish typeof-import type queries
- * from runtime calls instead of discarding the type reading on static precedence.
+ * Repair demonstrated syntax omissions in pinned 0.23.2 without changing source positions.
+ * Keep contextual names and type-only forms distinct from runtime calls; retain
+ * expression ambiguity through explicit GLR conflicts instead of guessing.
  * Source: https://github.com/tree-sitter/tree-sitter-typescript/tree/v0.23.2
  * MIT; CLI 0.24.4 uses emscripten/emsdk:3.1.64 for the WASM build.
  */
@@ -289,12 +374,40 @@ function rebuildTypescript(dir: string, work: string, only?: string): void {
   for (const [before, after] of [
     [
       '      _reserved_identifier: (_, previous) => choice(',
-      "      _reserved_identifier: (_, previous) => choice(\n        'unique',",
+      "      _reserved_identifier: (_, previous) => choice(\n        'unique',\n        'as',",
     ],
     ['      [$.call_expression, $._type_query_call_expression],\n', ''],
     [
       '    conflicts: ($, previous) => previous.concat([',
-      '    conflicts: ($, previous) => previous.concat([\n      [$.call_expression, $._type_query_call_expression],',
+      '    conflicts: ($, previous) => previous.concat([\n      [$.call_expression, $._type_query_call_expression],\n      [$.primary_expression, $.arrow_function, $.method_definition],',
+    ],
+    [
+      "          field('type_arguments', optional($.type_arguments)),\n          field('arguments', $.arguments),",
+      "          field('type_arguments', optional($.type_arguments)),\n          field('arguments', choice($.arguments, alias($._tagged_template_string, $.template_string))),",
+    ],
+    [
+      "      type_parameter: $ => seq(\n        optional('const'),\n        field('name', $._type_identifier),",
+      "      type_parameter: $ => seq(\n        optional(choice('const', 'in', 'out', seq('in', 'out'))),\n        field('name', choice($._type_identifier, alias('out', $.type_identifier))),",
+    ],
+    [
+      '          $.export_clause,\n          optional($._from_clause),',
+      "          choice(\n            seq($.export_clause, optional($._from_clause)),\n            seq(choice('*', $.namespace_export), $._from_clause),\n          ),",
+    ],
+    [
+      "            choice(',', $._semicolon),\n            choice(",
+      "            choice(',', $._semicolon, $._function_signature_automatic_semicolon),\n            choice(",
+    ],
+    [
+      "      mapped_type_clause: $ => seq(\n        field('name', $._type_identifier),",
+      "      mapped_type_clause: $ => seq(\n        field('name', choice($._type_identifier, alias('type', $.type_identifier))),",
+    ],
+    [
+      '          alias($._template_chars, $.string_fragment),\n          $.template_type,',
+      '          alias($._template_chars, $.string_fragment),\n          $.escape_sequence,\n          $.template_type,',
+    ],
+    [
+      "          field('name', choice($.identifier, $._destructuring_pattern)),",
+      "          field('name', choice($.identifier, alias('as', $.identifier), $._destructuring_pattern)),",
     ],
   ] as const) {
     if (!grammar.includes(before)) throw new Error('TypeScript grammar patch no longer applies');
@@ -328,7 +441,36 @@ function rebuildTypescript(dir: string, work: string, only?: string): void {
         optional(seq('as', field('alias', $._module_export_name))),
       ),`
   );
+  grammar = grammar.replaceAll(
+    "field('arguments', $.template_string)",
+    "field('arguments', alias($._tagged_template_string, $.template_string))"
+  );
   writeFileSync(grammarPath, grammar);
+  const scannerPath = join(dir, 'common', 'scanner.h');
+  let scanner = readFileSync(scannerPath, 'utf8');
+  const before = "        case '<':\n";
+  if (!scanner.includes(before))
+    throw new Error('TypeScript generic-overload ASI patch no longer applies');
+  scanner = scanner
+    .replace(before, '')
+    .replace(
+      "        case '{':",
+      "        case '<':\n            // Only type-member/signature boundaries may precede a new generic overload.\n            return valid_symbols[FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON] && !valid_symbols[LOGICAL_OR];\n\n        case '{':"
+    );
+  const inBoundary =
+    '            if (!iswalpha(lexer->lookahead)) {\n                return false;\n            }';
+  if (!scanner.includes(inBoundary))
+    throw new Error('TypeScript in-property ASI patch no longer applies');
+  scanner = scanner.replace(
+    inBoundary,
+    '            if (!iswalpha(lexer->lookahead)) {\n                return !valid_symbols[LOGICAL_OR];\n            }'
+  );
+  const optionalTuple =
+    "lexer->lookahead == ':' || lexer->lookahead == ')' || lexer->lookahead == ','";
+  if (!scanner.includes(optionalTuple))
+    throw new Error('TypeScript optional-tuple patch no longer applies');
+  scanner = scanner.replace(optionalTuple, optionalTuple + " || lexer->lookahead == ']'");
+  writeFileSync(scannerPath, scanner);
   // Retain our type/enum declaration captures when the vendor loop copies the query.
   copyFileSync(join(WASM_DIR, 'typescript.tags.scm'), join(dir, 'queries', 'tags.scm'));
   const tools = join(work, 'typescript-build');
@@ -343,8 +485,10 @@ function rebuildTypescript(dir: string, work: string, only?: string): void {
     'tree-sitter-javascript@0.23.1',
   ]);
   const cli = join(tools, 'node_modules', '.bin', 'tree-sitter');
+  // CLI 0.24 mis-emits the complementary character class; spell out raw characters.
+  patchTaggedTemplates(join(tools, 'node_modules', 'tree-sitter-javascript', 'grammar.js'), false);
   for (const dialect of ['typescript', 'tsx']) {
-    if (only && only !== dialect) continue;
+    if (only && !only.split(',').includes(dialect)) continue;
     // Apply the JSX scanner and attribute repairs only to TSX.
     if (dialect === 'tsx') {
       patchJsxScanner(join(dir, 'common', 'scanner.h'));
@@ -364,6 +508,34 @@ function rebuildTypescript(dir: string, work: string, only?: string): void {
   }
 }
 
+/** Tagged templates retain invalid escape spellings as raw text; ordinary literals stay strict. */
+function patchTaggedTemplates(grammarPath: string, complementaryClass = true): void {
+  let grammar = readFileSync(grammarPath, 'utf8');
+  const argument = "field('arguments', $.template_string)";
+  const template = '    template_string: $ => seq(';
+  if (!grammar.includes(argument) || !grammar.includes(template))
+    throw new Error('Tagged-template escape patch no longer applies');
+  const tagged = [
+    '    _tagged_template_string: $ => seq(',
+    "      '`',",
+    '      repeat(choice(',
+    '        alias($._template_chars, $.string_fragment),',
+    '        $.escape_sequence,',
+    complementaryClass
+      ? "        alias(token.immediate(prec(1, seq('\\\\', /[\\s\\S]/))), $.escape_sequence),"
+      : "        alias(token.immediate(prec(1, seq('\\\\', choice(/./, /[\\r\\n\\u2028\\u2029]/)))), $.escape_sequence),",
+    '        $.template_substitution,',
+    '      )),',
+    "      '`',",
+    '    ),',
+    '',
+  ].join('\n');
+  grammar = grammar
+    .replaceAll(argument, "field('arguments', alias($._tagged_template_string, $.template_string))")
+    .replace(template, tagged + template);
+  writeFileSync(grammarPath, grammar);
+}
+
 /** Upstream f8637eb: preserve entity boundaries while accepting bare JSX ampersands. */
 function patchJsxScanner(scannerPath: string): void {
   execFileSync('patch', ['--batch', scannerPath], {
@@ -394,6 +566,7 @@ function patchJsxAttributes(grammarPath: string): void {
 
 /** 0.25.0's reserved-word DSL needs generator 0.25.8; keep the existing WASM build tool. */
 function rebuildJavascript(dir: string, work: string): void {
+  patchTaggedTemplates(join(dir, 'grammar.js'));
   patchJsxScanner(join(dir, 'src', 'scanner.c'));
   patchJsxAttributes(join(dir, 'grammar.js'));
   const generator = join(work, 'javascript-generator');
@@ -513,6 +686,18 @@ function vendorObjcpp(work: string, repository: string): string {
   ]);
   git(['checkout', '--quiet', '--detach', 'FETCH_HEAD']);
   const dir = join(root, 'packages', 'tree-sitter-objcpp');
+  const grammarPath = join(dir, 'grammar.js');
+  let grammar = readFileSync(grammarPath, 'utf8');
+  const before = '    _objc_implementation_item: $ => choice(\n';
+  if (!grammar.includes(before))
+    throw new Error('Objective-C++ macro-body patch no longer applies');
+  grammar = grammar.replace(
+    before,
+    "    objc_macro_body: $ => seq($.objc_macro_invocation, field('body', $.compound_statement)),\n\n" +
+      before +
+      '      $.objc_macro_body,\n'
+  );
+  writeFileSync(grammarPath, grammar);
   const tools = join(work, 'objcpp-build');
   execFileSync('npm', [
     'install',
@@ -522,10 +707,17 @@ function vendorObjcpp(work: string, repository: string): string {
     '--no-fund',
     '--no-package-lock',
     'tree-sitter-cli@0.24.4',
+    // Match the pinned source's lockfile; these are grammar sources, never runtime dependencies.
+    'tree-sitter-c@0.23.6',
+    'tree-sitter-cpp@0.23.4',
   ]);
-  // Upstream generated this ABI-15 parser with CLI 0.25.10; 0.24.4 builds it with emsdk 3.1.64.
+  const cli = join(tools, 'node_modules', '.bin', 'tree-sitter');
+  execFileSync(cli, ['generate'], {
+    cwd: dir,
+    env: { ...process.env, NODE_PATH: join(tools, 'node_modules') },
+  });
   execFileSync(
-    join(tools, 'node_modules', '.bin', 'tree-sitter'),
+    cli,
     ['build', '--wasm', '--docker', '--output', join(dir, 'tree-sitter-objcpp.wasm')],
     { cwd: dir }
   );
@@ -611,12 +803,35 @@ function rebuildSql(dir: string, work: string): void {
       $.keyword_create, make_keyword("policy"), field('name', $.identifier),
       $.keyword_on, alias($.policy_relation, $.relation),
       optional(seq($.keyword_for, choice($.keyword_select, $.keyword_insert, $.keyword_update, $.keyword_delete))),
-      optional(seq($.keyword_to, $.identifier)),
+      optional(seq($.keyword_to, comma_list($.identifier, true))),
       optional(seq($.keyword_using, wrapped_in_parenthesis($._expression))),
       optional(seq($.keyword_with, $.keyword_check, wrapped_in_parenthesis($._expression))),
     ),
     create_table: $ => prec.left(`
   );
+  // These stored migrations use qualified enum names and table/sequence grants.
+  patch(
+    `    alter_type: $ => seq(
+      $.keyword_alter,
+      $.keyword_type,
+      $.identifier,`,
+    `    alter_type: $ => seq(
+      $.keyword_alter,
+      $.keyword_type,
+      $.object_reference,`
+  );
+  patch('      $.reset_statement,', '      $.reset_statement,\n      $.grant_statement,');
+  patch(
+    '    _cte: $ => seq(',
+    `    grant_statement: $ => seq(
+      make_keyword("grant"),
+      comma_list(choice($.keyword_select, $.keyword_insert, $.keyword_update, $.keyword_delete, make_keyword("usage")), true),
+      $.keyword_on, optional($.keyword_sequence), comma_list($.object_reference, true),
+      $.keyword_to, comma_list($.identifier, true),
+    ),
+
+    _cte: $ => seq(`
+  );
   // Dollar-quoted procedural code is one literal, never a parsed SQL statement list.
   patch('      $.reset_statement,', '      $.reset_statement,\n      $.do_statement,');
   patch(
@@ -639,6 +854,240 @@ function rebuildSql(dir: string, work: string): void {
     '  conflicts: $ => [',
     '  conflicts: $ => [\n    [$._expression, $._inner_default_expression],\n    [$.create_function],'
   );
+  // Fixed R2 cohort: change individual upstream rules, retaining the scanner and AST names.
+  const patchRule = (name: string, before: string, after: string): void => {
+    const start = grammar.indexOf(`    ${name}:`);
+    if (start < 0) throw new Error(`SQL rule missing: ${name}`);
+    const next = grammar.slice(start + 1).search(/^    [a-zA-Z_]\w*\s*:/m);
+    const end = next < 0 ? grammar.length : start + 1 + next;
+    const rule = grammar.slice(start, end);
+    if (!rule.includes(before)) throw new Error(`SQL ${name} patch no longer applies`);
+    grammar = grammar.slice(0, start) + rule.replace(before, after) + grammar.slice(end);
+  };
+  patchRule('_type', 'prec.left(', 'prec.right(');
+  patchRule('array_size_definition', 'prec.left(', 'prec.right(');
+  patchRule(
+    '_type',
+    '$.keyword_geography,',
+    `seq($.keyword_geography, optional(wrapped_in_parenthesis(
+            seq($.identifier, optional(seq(',', $._natural_number)))
+          ))),`
+  );
+  patchRule(
+    '_column_constraint',
+    '$.keyword_auto_increment,',
+    `choice($.keyword_auto_increment, make_keyword("autoincrement")),`
+  );
+  patchRule('function_argument', '$.literal,', '$._expression,');
+  patchRule(
+    '_inner_default_expression',
+    '$.keyword_current_timestamp,',
+    `$.keyword_current_timestamp,
+        make_keyword("current_date"),`
+  );
+  patchRule('create_policy', '$.keyword_delete))),', '$.keyword_delete, $.keyword_all))),');
+  patchRule(
+    '_create_statement',
+    '$.create_function,',
+    'alias($.create_procedure, $.create_function),\n        $.create_function,'
+  );
+  patchRule(
+    '_drop_statement',
+    '$.drop_function,',
+    '$.drop_function,\n        $.drop_procedure,\n        $.drop_policy,'
+  );
+  patchRule('_alter_statement', '$.alter_type,', '$.alter_type,\n        $.alter_policy,');
+  patchRule(
+    'drop_function',
+    '$.object_reference,',
+    '$.object_reference,\n      optional($.function_arguments),'
+  );
+  patchRule('drop_index', 'field("name", $.identifier)', 'field("name", $.object_reference)');
+  patchRule(
+    'alter_column',
+    "field('type', $._type),",
+    "field('type', $._type),\n          optional(seq($.keyword_using, $._expression)),"
+  );
+  patchRule(
+    'add_constraint',
+    '$.constraint,',
+    '$.constraint,\n      optional(seq($.keyword_not, make_keyword("valid"))),'
+  );
+  patchRule(
+    '_alter_specifications',
+    '$.set_schema,',
+    `$.set_schema,
+      seq(make_keyword("validate"), $.keyword_constraint, $.identifier),
+      seq($.keyword_rename, $.keyword_constraint, $.identifier, $.keyword_to, $.identifier),
+      seq($.keyword_set, paren_list(seq($.identifier, '=', $.literal), true)),
+      seq($.keyword_reset, paren_list($.identifier, true)),`
+  );
+  patchRule(
+    '_table_settings',
+    '$.table_partition,',
+    `seq($.keyword_on, $.keyword_commit, $.keyword_drop),
+      $.table_partition,`
+  );
+  patchRule(
+    'create_index',
+    '$.index_fields',
+    `$.index_fields,
+        optional(seq(make_keyword("include"), paren_list($.identifier, true)))`
+  );
+  patchRule(
+    'transaction',
+    "      optional(';'),",
+    `      optional(choice(
+        seq($.keyword_read, $.keyword_only),
+        seq($.keyword_isolation, $.keyword_level, $.keyword_repeatable, $.keyword_read)
+      )),
+      optional(';'),`
+  );
+  patchRule(
+    'limit',
+    '$.literal,',
+    'choice($.literal, $.parameter, alias($._qualified_field, $.field)),'
+  );
+  patchRule(
+    'offset',
+    '$.literal,',
+    'choice($.literal, $.parameter, alias($._qualified_field, $.field)),'
+  );
+  patchRule(
+    'statement',
+    `        optional($.keyword_analyze),
+        optional($.keyword_verbose),`,
+    `        choice(
+          seq(optional($.keyword_analyze), optional($.keyword_verbose)),
+          paren_list(seq(choice($.identifier, $.keyword_analyze, $.keyword_verbose),
+            optional(choice($.keyword_on, $.keyword_off, $.keyword_true, $.keyword_false))), true)
+        ),`
+  );
+  patchRule(
+    '_ddl_statement',
+    '$.reset_statement,',
+    `$.reset_statement,
+      $.revoke_statement,
+      $.alter_default_privileges,
+      $.analyze_statement,
+      $.lock_statement,
+      $.prepare_statement,
+      $.execute_statement,
+      $.call_statement,`
+  );
+  patchRule(
+    '_delete_from',
+    '$.object_reference,',
+    `$.object_reference,
+      optional($._alias),
+      optional(seq($.keyword_using, comma_list($.relation, true))),`
+  );
+  patchRule(
+    'cte',
+    'choice($._dml_read, $._dml_write)',
+    'choice($._dml_read, $._dml_write, $.values)'
+  );
+  patchRule(
+    '_on_conflict',
+    'optional($._column_list),',
+    'optional(seq(paren_list($._expression, true), optional($.where))),'
+  );
+  patchRule('from', `comma_list($.relation, true),`, '$.relation,');
+  patchRule(
+    'from',
+    `          $.join,`,
+    `          seq(',', $.relation),
+          $.join,`
+  );
+  patchRule('from', '      optional($.order_by),\n      optional($.limit),', '');
+  patchRule(
+    '_select_statement',
+    '        optional($.from),',
+    `        optional($.from),
+        optional($.order_by),
+        optional($.limit),`
+  );
+  patchRule(
+    'set_operation',
+    `      ),\n    ),`,
+    `      ),
+      optional($.order_by),
+      optional($.limit),
+    ),`
+  );
+  patchRule('op_other', "        '<->',", "        '<->',\n        '<=>',");
+  patchRule('_expression', '$.interval,', '$.interval,\n        $.at_time_zone,');
+  const support = '          $.function_support,\n';
+  if (grammar.split(support).length !== 3) throw new Error('SQL function options changed');
+  grammar = grammar.replaceAll(support, support + '          $.function_configuration,\n');
+  patchRule(
+    'grant_statement',
+    `comma_list(choice($.keyword_select, $.keyword_insert, $.keyword_update, $.keyword_delete, make_keyword("usage")), true),
+      $.keyword_on, optional($.keyword_sequence), comma_list($.object_reference, true),`,
+    `$._privilege_list,
+      $.keyword_on, $._privilege_target,`
+  );
+  patch(
+    '    _cte: $ => seq(',
+    `    _privilege_list: $ => comma_list(choice(
+      $.keyword_select, $.keyword_insert, $.keyword_update, $.keyword_delete,
+      make_keyword("usage"), make_keyword("execute"), make_keyword("truncate"),
+      seq($.keyword_all, optional(make_keyword("privileges")))
+    ), true),
+    routine_signature: $ => seq($.object_reference, optional($.function_arguments)),
+    _privilege_target: $ => choice(
+      seq($.keyword_function, comma_list($.routine_signature, true)),
+      seq($.keyword_schema, comma_list($.identifier, true)),
+      seq($.keyword_all, choice(make_keyword("tables"), make_keyword("sequences"), make_keyword("functions")),
+        $.keyword_in, $.keyword_schema, comma_list($.identifier, true)),
+      seq(optional(choice($.keyword_table, $.keyword_sequence)), comma_list($.object_reference, true))
+    ),
+    revoke_statement: $ => seq(make_keyword("revoke"), $._privilege_list, $.keyword_on,
+      $._privilege_target, $.keyword_from, comma_list($.identifier, true)),
+    alter_default_privileges: $ => seq(
+      $.keyword_alter, $.keyword_default, make_keyword("privileges"),
+      optional(seq($.keyword_for, $.keyword_role, comma_list($.identifier, true))),
+      optional(seq($.keyword_in, $.keyword_schema, comma_list($.identifier, true))),
+      choice(
+        seq(make_keyword("grant"), $._privilege_list, $.keyword_on,
+          choice(make_keyword("tables"), make_keyword("sequences"), make_keyword("functions")),
+          $.keyword_to, comma_list($.identifier, true)),
+        seq(make_keyword("revoke"), $._privilege_list, $.keyword_on,
+          choice(make_keyword("tables"), make_keyword("sequences"), make_keyword("functions")),
+          $.keyword_from, comma_list($.identifier, true))
+      )
+    ),
+    drop_policy: $ => seq($.keyword_drop, make_keyword("policy"), optional($._if_exists),
+      $.identifier, $.keyword_on, alias($.policy_relation, $.relation)),
+    alter_policy: $ => seq($.keyword_alter, make_keyword("policy"), $.identifier,
+      $.keyword_on, alias($.policy_relation, $.relation), $.keyword_rename, $.keyword_to, $.identifier),
+    function_configuration: $ => seq($.keyword_set, $.object_reference,
+      choice(seq($.keyword_from, make_keyword("current")),
+        seq(choice('=', $.keyword_to), comma_list(choice($.literal, $.identifier), true)))),
+    _procedure_option: $ => choice($.function_security, $.function_configuration),
+    create_procedure: $ => seq(
+      $.keyword_create, optional($._or_replace), $.keyword_procedure,
+      $.object_reference, $.function_arguments, repeat($._procedure_option),
+      choice(
+        seq($.function_language, repeat($._procedure_option), alias($.opaque_function_body, $.function_body)),
+        seq(alias($.opaque_function_body, $.function_body), repeat($._procedure_option), $.function_language)
+      ), repeat($._procedure_option)
+    ),
+    drop_procedure: $ => seq($.keyword_drop, $.keyword_procedure, optional($._if_exists),
+      $.routine_signature, optional($._drop_behavior)),
+    call_statement: $ => seq(make_keyword("call"), $.invocation),
+    analyze_statement: $ => seq($.keyword_analyze, $.object_reference, optional(paren_list($.identifier, true))),
+    lock_statement: $ => seq(make_keyword("lock"), $.keyword_table, comma_list($.object_reference, true),
+      $.keyword_in, choice(seq(make_keyword("share"), $.keyword_row, make_keyword("exclusive")),
+        seq(make_keyword("access"), make_keyword("exclusive"))), make_keyword("mode")),
+    prepare_statement: $ => seq(make_keyword("prepare"), $.identifier,
+      optional(paren_list($._type, true)), $.keyword_as, choice($._dml_read, $._dml_write)),
+    execute_statement: $ => seq(make_keyword("execute"), $.identifier, optional(paren_list($._expression))),
+    at_time_zone: $ => prec.left('binary_times', seq($._expression,
+      make_keyword("at"), make_keyword("time"), make_keyword("zone"), $._expression)),
+
+    _cte: $ => seq(`
+  );
   const functionStart = grammar.indexOf('    create_function: $ => seq(');
   const functionEnd = grammar.indexOf('    _function_return:', functionStart);
   if (functionStart < 0 || functionEnd < 0) throw new Error('SQL function patch no longer applies');
@@ -952,9 +1401,95 @@ function rebuildBash(dir: string, work: string): void {
   );
 }
 
+/** Reuse the existing offline WASM toolchain; never install native grammar bindings. */
+function buildPrerequisite(dir: string, work: string, spec: GrammarSpec): void {
+  const config = join(dir, 'tree-sitter.json');
+  const buildRepositories: Record<string, string> = {
+    clojure: 'https://github.com/oakmac/tree-sitter-clojure',
+    zig: 'https://github.com/GrayJack/tree-sitter-zig',
+    graphql: 'https://github.com/bkegley/tree-sitter-graphql',
+  };
+  if (!existsSync(config) && buildRepositories[spec.id]) {
+    // Old packages omit the build metadata required by the current WASM builder.
+    writeFileSync(
+      config,
+      JSON.stringify({
+        grammars: [
+          {
+            name: spec.id === 'protobuf' ? 'proto' : spec.id,
+            scope: `source.${spec.id}`,
+            path: '.',
+            'file-types': [],
+          },
+        ],
+        metadata: {
+          version: (
+            JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { version: string }
+          ).version,
+          license: spec.id === 'zig' ? 'BSD-3-Clause' : 'MIT',
+          description: 'Pinned parser WASM build',
+          authors: [],
+          links: { repository: buildRepositories[spec.id] },
+        },
+        bindings: {},
+      })
+    );
+  }
+  if (spec.id === 'graphql') {
+    // The pinned quoted-string rule rejects every escape, including valid GraphQL \n.
+    // Keep the upstream block-string rule and all source offsets unchanged.
+    const path = join(dir, 'grammar.js');
+    const grammar = readFileSync(path, 'utf8');
+    const before = String.raw`seq('"', /[^"\\\n]*/, '"')`;
+    const after = String.raw`seq('"', /([^"\\\r\n]|\\(["\\/bfnrt]|u[0-9a-fA-F]{4}))*/, '"')`;
+    if (!grammar.includes(before)) throw new Error('GraphQL string escape patch no longer applies');
+    writeFileSync(path, grammar.replace(before, after));
+  }
+  if (spec.id === 'zig') {
+    const path = join(dir, 'grammar.js');
+    const grammar = readFileSync(path, 'utf8');
+    const before = '/u{[0-9a-fA-F]+}/';
+    if (!grammar.includes(before))
+      throw new Error('Zig Unicode escape compatibility patch no longer applies');
+    writeFileSync(path, grammar.replace(before, String.raw`/u\{[0-9a-fA-F]+\}/`));
+  }
+  const tools = join(work, 'prerequisite-build');
+  const cli = join(tools, 'node_modules', '.bin', 'tree-sitter');
+  if (!existsSync(cli)) {
+    execFileSync('npm', [
+      'install',
+      '--prefix',
+      tools,
+      '--no-audit',
+      '--no-fund',
+      '--no-package-lock',
+      'tree-sitter-cli@0.24.4',
+    ]);
+  }
+  const deadline = Date.now() + 300_000;
+  if (spec.build === 'regenerate') {
+    execFileSync(cli, ['generate'], { cwd: dir, timeout: 300_000 });
+  }
+  execFileSync(
+    cli,
+    [
+      'build',
+      '--wasm',
+      '--docker',
+      '--output',
+      join(dir, spec.wasmFile ?? `tree-sitter-${spec.id}.wasm`),
+    ],
+    {
+      cwd: dir,
+      timeout: Math.max(1, deadline - Date.now()),
+    }
+  );
+}
+
 function vendor(): Problem[] {
   const only = process.argv.find((arg) => arg.startsWith('--only='))?.slice(7);
-  if (only && !GRAMMARS.some((spec) => spec.id === only))
+  const selected = only?.split(',');
+  if (selected?.some((id) => !GRAMMARS.some((spec) => spec.id === id)))
     throw new Error(`Unknown grammar: ${only}`);
   mkdirSync(WASM_DIR, { recursive: true });
   const work = mkdtempSync(join(tmpdir(), 'ts-grammars-'));
@@ -965,7 +1500,7 @@ function vendor(): Problem[] {
 
   try {
     for (const spec of GRAMMARS) {
-      if (only && spec.id !== only) continue;
+      if (selected && !selected.includes(spec.id)) continue;
       let dir = extracted.get(spec.pkg);
       if (!dir) {
         try {
@@ -998,6 +1533,7 @@ function vendor(): Problem[] {
             if (spec.id === 'cpp') rebuildCpp(dir, work);
             if (spec.id === 'bash') rebuildBash(dir, work);
             if (spec.id === 'nix' || spec.id === 'cmake') rebuildNixOrCmake(dir, work, spec.id);
+            if (spec.build) buildPrerequisite(dir, work, spec);
           }
           extracted.set(spec.pkg, dir);
         } catch (err) {
@@ -1015,7 +1551,18 @@ function vendor(): Problem[] {
       if (!existsSync(wasmSrc)) {
         problems.push({ id: spec.id, detail: `${spec.pkg} ships no ${wasmName}` });
       } else {
-        copyFileSync(wasmSrc, join(WASM_DIR, `${spec.id}.wasm`));
+        const compressed = gzipSync(readFileSync(wasmSrc), { level: 9 });
+        // Timestamp is zero in Node gzip; normalize the OS byte across build hosts too.
+        compressed[9] = 255;
+        writeFileSync(join(WASM_DIR, `${spec.id}.wasm.gz`), compressed, { mode: 0o644 });
+        rmSync(join(WASM_DIR, `${spec.id}.wasm`), { force: true });
+        if (spec.licenseFile)
+          writeFileSync(
+            join(WASM_DIR, `${spec.id}.LICENSE`),
+            readFileSync(join(dir, spec.licenseFile), 'utf8')
+              .replace(/\r\n/g, '\n')
+              .replace(/[\u201c\u201d]/g, '"')
+          );
         if (spec.id === 'lua') copyFileSync(join(dir, 'LICENSE.md'), join(WASM_DIR, 'lua.LICENSE'));
         if (spec.id === 'terraform')
           copyFileSync(join(dir, 'LICENSE'), join(WASM_DIR, 'terraform.LICENSE'));
@@ -1029,7 +1576,7 @@ function vendor(): Problem[] {
           spec.id === 'nix' ||
           spec.id === 'cmake'
         )
-          chmodSync(join(WASM_DIR, `${spec.id}.wasm`), 0o644);
+          chmodSync(join(WASM_DIR, `${spec.id}.wasm.gz`), 0o644);
       }
 
       // A targeted JS scanner rebuild must retain our module-scope binding query additions.
@@ -1072,11 +1619,15 @@ if (problems.length > 0) {
   process.exit(1);
 }
 
-const sizes = GRAMMARS.map((g) => statSync(join(WASM_DIR, `${g.id}.wasm`)).size);
+const sizes = GRAMMARS.map((g) => statSync(join(WASM_DIR, `${g.id}.wasm.gz`)).size);
 const totalBytes = sizes.reduce((a, b) => a + b, 0);
+const inflatedBytes = GRAMMARS.reduce(
+  (total, g) => total + gunzipSync(readFileSync(join(WASM_DIR, `${g.id}.wasm.gz`))).length,
+  0
+);
 const totalMB = +(totalBytes / 1048576).toFixed(1);
 console.log(
-  `VENDOR_GRAMMARS_OK ${JSON.stringify({ grammars: GRAMMARS.length, totalWasmMB: totalMB })}`
+  `VENDOR_GRAMMARS_OK ${JSON.stringify({ grammars: GRAMMARS.length, shippedBytes: totalBytes, inflatedBytes, totalWasmMB: totalMB })}`
 );
 // Owner-approved Objective-C++ cost: repaired bundle 43.58 MiB (43.67 with Terraform 667475e93), bounded at 45 MiB.
 if (totalBytes > 45 * 1048576) {