@tabnas/proto 0.2.3 → 0.3.0

package/README.md

@@ -54,14 +54,30 @@ reconcile?: boolean }`.
 ## What it produces
 
 A `FileDescriptorProto`-shaped object (the `descriptor.proto` JSON shape):
-`package`, `dependency` (+ `publicDependency` / `weakDependency`),
-`messageType` (recursive `DescriptorProto` with `field`, `nestedType`,
-`enumType`, `oneofDecl`, `extensionRange`, `reservedRange`), `enumType`,
-`service`, `extension`, `options`, and `syntax` / `edition`. `map<K,V>`
-fields are expanded to a repeated message field plus a synthesised
-`…Entry` nested message with `options.mapEntry = true`, exactly as `protoc`
-does. Type names are stored as written; cross-file resolution is a separate
-concern.
+`package`, `dependency` (+ `publicDependency` / `weakDependency` /
+`optionDependency`), `messageType` (recursive `DescriptorProto` with
+`field`, `nestedType`, `enumType`, `oneofDecl`, `extensionRange`,
+`reservedRange`), `enumType`, `service`, `extension`, `options`, and
+`syntax` / `edition`. `map<K,V>` fields are expanded to a repeated message
+field plus a synthesised `…Entry` nested message with
+`options.mapEntry = true`, groups to a `TYPE_GROUP` field plus a nested
+message, and a proto3 explicit `optional` to a synthetic `_<field>` oneof —
+exactly as `protoc` does. Type names are stored as written and `type` is
+left unset for them; cross-file resolution is a separate concern.
+
+Two things are deliberately shaped for readability rather than byte-for-byte
+`protoc` parity: options are a plain `{ name: value }` map (not an
+`uninterpretedOption` list), and `defaultValue` keeps the literal as
+written. See [doc/reference.md](doc/reference.md).
+
+## Conformance
+
+Checked against protoc 35.1's own parser test corpus (extracted from
+`parser_unittest.cc`): every one of the 71 in-scope `valid` cases and all
+50 `accept-only` cases, run by `test/protobuf-conformance.test.ts`. The 11
+excluded cases declare protoc-internal editions (`UNSTABLE`,
+`99998_TEST_ONLY`) outside the proto2 / proto3 / 2023 / 2024 support this
+package claims.
 
 See [doc/tutorial.md](doc/tutorial.md), [doc/guide.md](doc/guide.md),
 [doc/reference.md](doc/reference.md), and [doc/concepts.md](doc/concepts.md).

package/dist/build-descriptor.js

@@ -10,9 +10,11 @@ exports.buildFile = buildFile;
 // kind from the keyword that precedes the node's first child (`kw`).
 const descriptor_1 = require("./descriptor");
 const detect_version_1 = require("./detect-version");
-// Children that are real rule nodes (terminals fold into `src`).
+// Children that are real rule nodes (terminals fold into `src`). An empty
+// `.proto` is a legal (if useless) file and parses to no node at all, so
+// tolerate a missing node here rather than crash at the root.
 function R(n) {
-    return (n.kids || []).filter((k) => k && k.rule);
+    return ((n && n.kids) || []).filter((k) => k && k.rule);
 }
 // The keyword(s) consumed before this node's first child — the part of
 // `src` ahead of the first child's `src`. For `message Foo {…}` the first
@@ -45,31 +47,66 @@ function constantValue(n) {
         if (!Number.isNaN(num))
             return num;
     }
-    return s; // identifier (enum value name, inf, nan, …) kept verbatim
+    return s; // identifier (enum value name, inf, -nan, …) kept verbatim
 }
-// The option name in an `optionName "=" constant` statement. Because the
-// name's leading `optionNamePart` is inlined and the parts are split across
-// the node's `src` and later children, read it as everything before the
-// `constant`, with the trailing `=` removed.
+// The option name in an `optionName "=" constant` statement.
+//
+// `optionStmt` keeps `optionName` as a child node, so use it when present.
+// Inside `fieldOption` abnf inlines the leading `optionNamePart`, so there
+// the name has to be read out of `src` — as everything BEFORE the trailing
+// `"=" constant`. Anchoring at the end matters: `option (file_opt1) = 1;`
+// would otherwise find the value `1` inside the name `file_opt1`.
 function optionNameOf(stmt, valueNode) {
+    const named = child(stmt, 'optionName');
+    if (named)
+        return named.src;
     if (!valueNode)
         return '';
-    const i = stmt.src.indexOf(valueNode.src);
-    return (i <= 0 ? '' : stmt.src.slice(0, i)).replace(/=$/, '');
+    const s = stmt.src.replace(/;$/, '');
+    const tail = '=' + valueNode.src;
+    if (s.endsWith(tail))
+        return s.slice(0, s.length - tail.length);
+    const i = s.indexOf(valueNode.src);
+    return i <= 0 ? '' : s.slice(0, i).replace(/=$/, '');
 }
 // fieldOptions = "[" fieldOption *( "," fieldOption ) "]"
 // fieldOption  = optionName "=" constant
 function readFieldOptions(opts) {
-    if (!opts)
-        return undefined;
     const out = {};
+    if (!opts)
+        return out;
+    const map = {};
     for (const fo of R(opts)) {
         const cst = child(fo, 'constant');
         const name = optionNameOf(fo, cst);
-        if (cst && name)
-            out[name] = constantValue(cst);
+        if (!cst || !name)
+            continue;
+        if ('json_name' === name) {
+            out.jsonName = String(unquote(cst.src));
+            continue;
+        }
+        if ('default' === name) {
+            out.defaultValue = unquote(cst.src);
+            continue;
+        }
+        map[name] = constantValue(cst);
     }
-    return Object.keys(out).length ? out : undefined;
+    if (Object.keys(map).length)
+        out.options = map;
+    return out;
+}
+// The plain (non-pseudo) option map, for the places that cannot carry
+// `json_name` / `default` — extension ranges, enum values.
+function plainOptions(opts) {
+    return readFieldOptions(opts).options;
+}
+// optionStmt = "option" optionName "=" constant ";"
+function optionFrom(el) {
+    const cst = child(el, 'constant');
+    if (!cst)
+        return {};
+    const name = optionNameOf(el, cst);
+    return name ? { [name]: constantValue(cst) } : {};
 }
 // ---- fields --------------------------------------------------------------
 function fieldLabel(labelNode, version) {
@@ -91,8 +128,11 @@ function fieldTypeName(typeText) {
     const scalar = descriptor_1.SCALAR_TYPES[bare];
     if (scalar)
         return { type: scalar };
-    // Message or enum reference; resolution deferred, store as written.
-    return { type: 'TYPE_MESSAGE', typeName: typeText };
+    // A named reference: could be a message OR an enum, and telling them
+    // apart needs symbol resolution this parser deliberately does not do.
+    // protoc leaves `type` unset here too, and only fills it in once the
+    // name resolves — so record `typeName` as written and nothing else.
+    return { typeName: typeText };
 }
 // A normal field: [label] fieldType ident "=" fieldNumber [fieldOptions].
 // The surrounding dispatch node (`messageElement` / `oneofElement`) has these
@@ -103,23 +143,79 @@ function fieldTypeName(typeText) {
 function typeNodeOf(n) {
     return child(n, 'fieldType') ?? child(n, 'messageType') ?? child(n, 'fullIdent');
 }
+function applyFieldOptions(f, opts) {
+    const po = readFieldOptions(opts);
+    if (undefined !== po.jsonName)
+        f.jsonName = po.jsonName;
+    if (undefined !== po.defaultValue)
+        f.defaultValue = po.defaultValue;
+    if (po.options)
+        f.options = po.options;
+}
 function buildField(n, version) {
     const label = child(n, 'label');
     const type = typeNodeOf(n);
     const name = child(n, 'ident');
     const number = child(n, 'fieldNumber');
-    const opts = child(n, 'fieldOptions');
     const f = {
         name: name ? name.src : '',
         number: number ? Number(number.src) : 0,
         ...fieldLabel(label, version),
         ...fieldTypeName(type ? type.src : ''),
     };
-    const fo = readFieldOptions(opts);
-    if (fo)
-        f.options = fo;
+    applyFieldOptions(f, child(n, 'fieldOptions'));
+    return f;
+}
+// A group is a field plus an implicit nested message. protoc lowercases the
+// field name and keeps the declared name for the message and for `typeName`.
+//   `optional group TestGroup = 1 { … }`
+//     -> field { name: 'testgroup', type: TYPE_GROUP, typeName: 'TestGroup' }
+//     -> nestedType { name: 'TestGroup', … }
+function buildGroup(n, version, into) {
+    const groupName = child(n, 'ident')?.src ?? '';
+    const number = child(n, 'fieldNumber');
+    into.nestedType.push(messageFromBody(groupName, child(n, 'messageBody'), version));
+    const f = {
+        name: groupName.toLowerCase(),
+        number: number ? Number(number.src) : 0,
+        ...fieldLabel(child(n, 'label'), version),
+        type: 'TYPE_GROUP',
+        typeName: groupName,
+    };
+    applyFieldOptions(f, child(n, 'fieldOptions'));
     return f;
 }
+// A group is the one construct with BOTH a field number and a message body;
+// `message` has a body but no number, `map`/`field` a number but no body.
+function isGroup(n) {
+    return !!child(n, 'fieldNumber') && !!child(n, 'messageBody');
+}
+// The subset of an option map whose names are rooted at `features`.
+function features(opts) {
+    if (!opts)
+        return undefined;
+    const out = {};
+    for (const [k, v] of Object.entries(opts)) {
+        if ('features' === k || k.startsWith('features.'))
+            out[k] = v;
+    }
+    return Object.keys(out).length ? out : undefined;
+}
+// protoc's map-entry name: strip `_`, upper-case the letter that follows
+// (and the first letter), then append `Entry`. `map_field` -> `MapFieldEntry`.
+function mapEntryName(fieldName) {
+    let out = '';
+    let capNext = true;
+    for (const ch of fieldName) {
+        if ('_' === ch) {
+            capNext = true;
+            continue;
+        }
+        out += capNext && 'a' <= ch && ch <= 'z' ? ch.toUpperCase() : ch;
+        capNext = false;
+    }
+    return out + 'Entry';
+}
 // map<K,V> name = N;  ->  a repeated message field whose type is a
 // synthesised nested `<Name>Entry` message with mapEntry=true.
 function buildMapField(n, version, into) {
@@ -129,14 +225,12 @@ function buildMapField(n, version, into) {
     const keyText = types[0] ? types[0].src : '';
     const valText = types[1] ? types[1].src : '';
     const fname = name ? name.src : '';
-    const entryName = fname.replace(/(^|_)([a-z])/g, (_, a, b) => a + b.toUpperCase())
-        + 'Entry';
+    const entryName = mapEntryName(fname);
+    const key = { name: 'key', number: 1, label: 'LABEL_OPTIONAL', ...fieldTypeName(keyText) };
+    const value = { name: 'value', number: 2, label: 'LABEL_OPTIONAL', ...fieldTypeName(valText) };
     const entry = {
         name: entryName,
-        field: [
-            { name: 'key', number: 1, label: 'LABEL_OPTIONAL', ...fieldTypeName(keyText) },
-            { name: 'value', number: 2, label: 'LABEL_OPTIONAL', ...fieldTypeName(valText) },
-        ],
+        field: [key, value],
         nestedType: [], enumType: [], oneofDecl: [], extension: [],
         options: { mapEntry: true },
     };
@@ -145,12 +239,16 @@ function buildMapField(n, version, into) {
         name: fname,
         number: number ? Number(number.src) : 0,
         label: 'LABEL_REPEATED',
-        type: 'TYPE_MESSAGE',
         typeName: entryName,
     };
-    const fo = readFieldOptions(child(n, 'fieldOptions'));
-    if (fo)
-        f.options = fo;
+    applyFieldOptions(f, child(n, 'fieldOptions'));
+    // `features` on a map field govern the synthesised entry's key and value
+    // fields too, so protoc copies them down. Nothing else is propagated.
+    const feat = features(f.options);
+    if (feat) {
+        key.options = { ...feat };
+        value.options = { ...feat };
+    }
     return f;
 }
 // ---- enums ---------------------------------------------------------------
@@ -159,26 +257,29 @@ function buildEnum(n) {
     for (const el of R(n).filter((k) => k.rule === 'enumElement')) {
         const k = kw(el);
         if (k.startsWith('reserved')) {
-            addReserved(el, e);
+            // Enum reserved ranges are INCLUSIVE and span the whole int32 space.
+            addReserved(el, e, { exclusive: false, max: descriptor_1.MAX_ENUM_NUMBER });
             continue;
         }
         if (k.startsWith('option')) {
-            continue; // enum-level option; folded into options below if needed
+            e.options = { ...(e.options || {}), ...optionFrom(el) };
+            continue;
         }
         // enumField: ident "=" ["-"] fieldNumber  -> name is the kw before "="
-        const name = kw(el).replace(/=.*$/, '');
+        const name = k.replace(/=.*$/, '');
         const num = child(el, 'fieldNumber');
         if (name && num) {
             const neg = /=-/.test(el.src.replace(/\s+/g, ''));
-            e.value.push({ name, number: (neg ? -1 : 1) * Number(num.src) });
+            const v = { name, number: (neg ? -1 : 1) * Number(num.src) };
+            const vo = plainOptions(child(el, 'fieldOptions'));
+            e.value.push(vo ? { ...v, options: vo } : v);
         }
     }
     return e;
 }
-// ---- reserved / extensions ranges ----------------------------------------
 // `ranges = range *( "," range )`. The leading `range` is inlined into
 // `ranges.src`, so parse the (whitespace-stripped) text rather than kids.
-function ranges(rangesNode) {
+function ranges(rangesNode, ro) {
     if (!rangesNode)
         return [];
     const out = [];
@@ -187,38 +288,97 @@ function ranges(rangesNode) {
         if (!m)
             continue;
         const start = Number(m[1]);
-        const end = !m[2] ? start : (m[2] === 'max' ? 536870911 : Number(m[2]));
+        let end;
+        if (!m[2])
+            end = ro.exclusive ? start + 1 : start;
+        else if ('max' === m[2])
+            end = ro.max;
+        else
+            end = Number(m[2]) + (ro.exclusive ? 1 : 0);
         out.push({ start, end });
     }
     return out;
 }
-function addReserved(n, target) {
+// The reserved-name list. Both the leading `strLit`/`ident` and (for a
+// single-item list) the whole list can be inlined into `src`, so read the
+// names out of the statement text; whole-word tokens make that unambiguous.
+function reservedNames(n) {
+    const body = n.src.replace(/^reserved/, '').replace(/;$/, '');
+    const re = /"((?:[^"\\]|\\.)*)"|'((?:[^'\\]|\\.)*)'|([A-Za-z_][A-Za-z0-9_]*)/g;
+    const out = [];
+    let m;
+    while (null !== (m = re.exec(body)))
+        out.push(m[1] ?? m[2] ?? m[3]);
+    return out;
+}
+function addReserved(n, target, ro) {
     const rn = child(n, 'ranges');
     if (rn) {
-        (target.reservedRange = target.reservedRange || []).push(...ranges(rn));
+        (target.reservedRange = target.reservedRange || []).push(...ranges(rn, ro));
         return;
     }
-    const names = child(n, 'fieldNames');
-    if (names) {
-        const list = R(names).filter((k) => k.rule === 'strLit').map((k) => unquote(k.src));
-        (target.reservedName = target.reservedName || []).push(...list);
+    const names = reservedNames(n);
+    if (names.length) {
+        (target.reservedName = target.reservedName || []).push(...names);
     }
 }
 // ---- messages ------------------------------------------------------------
 function buildMessage(n, version) {
     // `n` is a dispatch node whose `message` alt was inlined: kids are
     // [ident, messageBody-children…] or [ident] then messageBody.
+    return messageFromBody(child(n, 'ident')?.src ?? '', child(n, 'messageBody'), version);
+}
+function messageFromBody(name, body, version) {
     const msg = {
-        name: child(n, 'ident')?.src ?? '',
+        name,
         field: [], nestedType: [], enumType: [], oneofDecl: [], extension: [],
     };
-    const body = child(n, 'messageBody');
     const elements = body ? R(body).filter((k) => k.rule === 'messageElement') : [];
-    for (const el of elements)
-        addMessageElement(el, version, msg);
+    // Options first: `message_set_wire_format` changes what `to max` means in
+    // an extension/reserved range, and protoc applies it wherever the option
+    // sits in the body.
+    for (const el of elements) {
+        if (isOptionStmt(el))
+            msg.options = { ...(msg.options || {}), ...optionFrom(el) };
+    }
+    const ro = {
+        exclusive: true,
+        max: true === msg.options?.message_set_wire_format
+            ? descriptor_1.MAX_MESSAGE_SET_END : descriptor_1.MAX_FIELD_NUMBER_END,
+    };
+    for (const el of elements) {
+        if (!isOptionStmt(el))
+            addMessageElement(el, version, msg, ro);
+    }
+    generateSyntheticOneofs(msg);
     return msg;
 }
-function addMessageElement(el, version, msg) {
+// An `option` statement, as opposed to an `optional`-labelled field (whose
+// `kw` is empty because the `label` node starts the src).
+function isOptionStmt(el) {
+    return kw(el).startsWith('option');
+}
+// protoc synthesises a single-field oneof for every proto3 explicit
+// `optional` field, appended after the declared oneofs. The name is the
+// field name prefixed with `_`, then prefixed with `X` until unique.
+function generateSyntheticOneofs(msg) {
+    const names = new Set();
+    for (const f of msg.field)
+        names.add(f.name);
+    for (const o of msg.oneofDecl)
+        names.add(o.name);
+    for (const f of msg.field) {
+        if (true !== f.proto3Optional || undefined !== f.oneofIndex)
+            continue;
+        let oneofName = '_' + f.name;
+        while (names.has(oneofName))
+            oneofName = 'X' + oneofName;
+        names.add(oneofName);
+        f.oneofIndex = msg.oneofDecl.length;
+        msg.oneofDecl.push({ name: oneofName });
+    }
+}
+function addMessageElement(el, version, msg, ro) {
     const k = kw(el);
     const first = R(el)[0];
     if (k.startsWith('map<')) {
@@ -231,12 +391,11 @@ function addMessageElement(el, version, msg) {
     }
     if (k.startsWith('export') || k.startsWith('local')) {
         // edition 2024 symbol visibility wraps the message/enum as a child node.
-        const m = child(el, 'message');
-        const e = child(el, 'enumDef');
-        if (m)
-            msg.nestedType.push(buildMessage(m, version));
-        else if (e)
-            msg.enumType.push(buildEnum(e));
+        addVisible(el, k, version, msg.nestedType, msg.enumType);
+        return;
+    }
+    if (isGroup(el)) {
+        msg.field.push(buildGroup(el, version, msg));
         return;
     }
     if (k.startsWith('message')) {
@@ -248,21 +407,24 @@ function addMessageElement(el, version, msg) {
         return;
     }
     if (k.startsWith('reserved')) {
-        addReserved(el, msg);
+        addReserved(el, msg, ro);
         return;
     }
     if (k.startsWith('extensions')) {
-        (msg.extensionRange = msg.extensionRange || []).push(...ranges(child(el, 'ranges')));
+        const opts = plainOptions(child(el, 'fieldOptions'));
+        const rs = ranges(child(el, 'ranges'), ro);
+        // A compound `extensions 2, 9 to 11 [(i) = 5];` puts the options on
+        // every range, as protoc does.
+        for (const r of rs)
+            if (opts)
+                r.options = { ...opts };
+        (msg.extensionRange = msg.extensionRange || []).push(...rs);
         return;
     }
     if (k.startsWith('extend')) {
         addExtend(el, version, msg.extension);
         return;
     }
-    if (k.startsWith('option')) {
-        msg.options = { ...(msg.options || {}), ...optionFrom(el) };
-        return;
-    }
     if (';' === el.src)
         return; // emptyStmt
     // No keyword and a fieldType/label lead => a field.
@@ -270,6 +432,17 @@ function addMessageElement(el, version, msg) {
         msg.field.push(buildField(el, version));
     }
 }
+// edition 2024 `export` / `local` on a message or enum declaration. The
+// wrapped `message` / `enumDef` stays a child node instead of inlining.
+function addVisible(el, k, version, messages, enums) {
+    const vis = k.startsWith('export') ? 'VISIBILITY_EXPORT' : 'VISIBILITY_LOCAL';
+    const m = child(el, 'message');
+    const e = child(el, 'enumDef');
+    if (m)
+        messages.push({ ...buildMessage(m, version), visibility: vis });
+    else if (e)
+        enums.push({ ...buildEnum(e), visibility: vis });
+}
 function addOneof(el, version, msg) {
     const name = child(el, 'ident')?.src ?? '';
     const index = msg.oneofDecl.length;
@@ -279,7 +452,7 @@ function addOneof(el, version, msg) {
             continue;
         if (';' === of.src)
             continue;
-        const f = buildField(of, version);
+        const f = isGroup(of) ? buildGroup(of, version, msg) : buildField(of, version);
         f.oneofIndex = index;
         delete f.proto3Optional; // explicit oneof members aren't proto3-optional
         msg.field.push(f);
@@ -287,25 +460,22 @@ function addOneof(el, version, msg) {
 }
 function addExtend(el, version, into) {
     // extend messageType "{" *field "}" — fields inline as messageElement-like.
+    const extendee = child(el, 'messageType')?.src;
     for (const f of R(el).filter((k) => k.rule === 'field' || k.rule === 'messageElement')) {
-        into.push(buildField(f, version));
+        const fd = buildField(f, version);
+        if (extendee)
+            fd.extendee = extendee;
+        into.push(fd);
     }
 }
-// ---- options -------------------------------------------------------------
-// optionStmt = "option" optionName "=" constant ";"
-function optionFrom(el) {
-    const cst = child(el, 'constant');
-    if (!cst)
-        return {};
-    const name = optionNameOf(el, cst).replace(/^option/, '');
-    return { [name]: constantValue(cst) };
-}
 // ---- services ------------------------------------------------------------
 function buildService(n) {
     const svc = { name: child(n, 'ident')?.src ?? '', method: [] };
     for (const el of R(n).filter((k) => k.rule === 'serviceElement')) {
         if (kw(el).startsWith('rpc'))
             svc.method.push(buildRpc(el));
+        else if (isOptionStmt(el))
+            svc.options = { ...(svc.options || {}), ...optionFrom(el) };
     }
     return svc;
 }
@@ -328,6 +498,9 @@ function buildRpc(el) {
         m.clientStreaming = true;
     if (/\(stream/.test(response))
         m.serverStreaming = true;
+    for (const o of R(el).filter((k) => k.rule === 'optionStmt')) {
+        m.options = { ...(m.options || {}), ...optionFrom(o) };
+    }
     return m;
 }
 // ---- file ----------------------------------------------------------------
@@ -336,10 +509,13 @@ function buildFile(proto, version) {
         dependency: [], publicDependency: [], weakDependency: [],
         messageType: [], enumType: [], service: [], extension: [],
     };
-    if ((0, detect_version_1.isEdition)(version))
+    if ((0, detect_version_1.isEdition)(version)) {
         file.edition = 'EDITION_' + version;
-    else
+        file.syntax = 'editions';
+    }
+    else {
         file.syntax = version;
+    }
     for (const def of R(proto).filter((k) => k.rule === 'topLevelDef')) {
         const k = kw(def);
         if (k.startsWith('package')) {
@@ -348,24 +524,25 @@ function buildFile(proto, version) {
         else if (k.startsWith('import')) {
             const s = child(def, 'strLit');
             if (s) {
-                const idx = file.dependency.length;
-                file.dependency.push(unquote(s.src));
-                if (k.includes('public'))
-                    file.publicDependency.push(idx);
-                if (k.includes('weak'))
-                    file.weakDependency.push(idx);
+                // `import option "x";` (edition 2024) is a separate dependency list.
+                if (k.includes('option')) {
+                    (file.optionDependency = file.optionDependency || []).push(unquote(s.src));
+                }
+                else {
+                    const idx = file.dependency.length;
+                    file.dependency.push(unquote(s.src));
+                    if (k.includes('public'))
+                        file.publicDependency.push(idx);
+                    if (k.includes('weak'))
+                        file.weakDependency.push(idx);
+                }
             }
         }
-        else if (k.startsWith('option')) {
+        else if (isOptionStmt(def)) {
             file.options = { ...(file.options || {}), ...optionFrom(def) };
         }
         else if (k.startsWith('export') || k.startsWith('local')) {
-            const m = child(def, 'message');
-            const e = child(def, 'enumDef');
-            if (m)
-                file.messageType.push(buildMessage(m, version));
-            else if (e)
-                file.enumType.push(buildEnum(e));
+            addVisible(def, k, version, file.messageType, file.enumType);
         }
         else if (k.startsWith('message')) {
             file.messageType.push(buildMessage(def, version));