@typespec/protobuf 0.43.0

package/lib/proto.tsp

@@ -0,0 +1,303 @@
+import "../dist/src/index.js";
+
+namespace TypeSpec.Protobuf;
+
+/**
+ * A model that represents an external Protobuf reference. This type can be used to import and utilize Protobuf
+ * declarations that are not declared in TypeSpec within TypeSpec sources. When the emitter encounters an `Extern`, it
+ * will insert an `import` statement for the corresponding `Path` and refer to the type by `Name`.
+ *
+ * #### Usage
+ *
+ * If you have a file called `test.proto` that declares a package named `test` and a message named `Widget`, you can
+ * use the `Extern` type to declare a model in TypeSpec that refers to your external definition of `test.Widget`. See
+ * the example below.
+ *
+ * When the TypeSpec definition of `Widget` is encountered, the Protobuf emitter will represent it as a reference to
+ * `test.Widget` and insert an import for it, rather than attempt to convert the model to an equivalent message.
+ *
+ * @template Path the relative path to a `.proto` file to import
+ * @template Name the fully-qualified reference to the type this model represents within the `.proto` file
+ *
+ * @example
+ *
+ * ```typespec
+ * model Widget is Extern<"path/to/test.proto", "test.Widget">;
+ * ```
+ */
+@externRef(Path, Name)
+model Extern<Path extends string, Name extends string> {
+  // This _extern property is needed so that getEffectiveModelType will have something to look up. Without it, if an
+  // Extern model is spread into the parameter of an operation, the resulting model is empty and carries no information
+  // that can relate it back to its original definition.
+  _extern: never;
+}
+
+/**
+ * Contains some common well-known Protobuf types defined by the google.protobuf library.
+ */
+namespace WellKnown {
+  /**
+   * An empty message.
+   *
+   * This model references `google.protobuf.Empty` from `google/protobuf/empty.proto`.
+   */
+  model Empty is Extern<"google/protobuf/empty.proto", "google.protobuf.Empty">;
+
+  /**
+   * A timestamp.
+   *
+   * This model references `google.protobuf.Timestamp` from `google/protobuf/timestamp.proto`.
+   */
+  model Timestamp is Extern<"google/protobuf/timestamp.proto", "google.protobuf.Timestamp">;
+
+  /**
+   * Any value.
+   *
+   * This model references `google.protobuf.Any` from `google/protobuf/any.proto`.
+   */
+  model Any is Extern<"google/protobuf/any.proto", "google.protobuf.Any">;
+
+  /**
+   * A latitude and longitude.
+   *
+   * This model references `google.type.LatLng` from `google/type/latlng.proto`.
+   */
+  model LatLng is Extern<"google/type/latlng.proto", "google.type.LatLng">;
+}
+
+/**
+ * A signed 32-bit integer that will use the `sint32` encoding when used in a Protobuf message.
+ *
+ * #### Protobuf binary format
+ *
+ * Uses variable-length encoding. These more efficiently encode negative numbers than regular int32s.
+ */
+scalar sint32 extends int32;
+/**
+ * A signed 64-bit integer that will use the `sint64` encoding when used in a Protobuf message.
+ *
+ * #### Protobuf binary format
+ *
+ * Uses variable-length encoding. These more efficiently encode negative numbers than regular `int64s`.
+ */
+scalar sint64 extends int64;
+/**
+ * A signed 32-bit integer that will use the `sfixed32` encoding when used in a Protobuf message.
+ *
+ * #### Protobuf binary format
+ *
+ * Always four bytes.
+ */
+scalar sfixed32 extends int32;
+/**
+ * A signed 64-bit integer that will use the `sfixed64` encoding when used in a Protobuf message.
+ *
+ * #### Protobuf binary format
+ *
+ * Always eight bytes.
+ */
+scalar sfixed64 extends int64;
+/**
+ * An unsigned 32-bit integer that will use the `fixed32` encoding when used in a Protobuf message.
+ *
+ * #### Protobuf binary format
+ *
+ * Always four bytes. More efficient than `uint32` if values are often greater than 2<sup>28</sup>.
+ */
+scalar fixed32 extends uint32;
+/**
+ * An unsigned 64-bit integer that will use the `fixed64` encoding when used in a Protobuf message.
+ *
+ * #### Protobuf binary format
+ *
+ * Always eight bytes. More efficient than `uint64` if values are often greater than 2<sup>56</sup>.
+ */
+scalar fixed64 extends uint64;
+
+/**
+ * Types recognized as "integral" types
+ */
+alias integral = int32 | int64 | uint32 | uint64 | boolean;
+
+/**
+ * A type representing a Protobuf `map`. Instances of this type in models will be converted to the built-in `map` type
+ * in Protobuf.
+ *
+ * The key type of a Protobuf `map` must be any integral type or `string`. The value type can be any type other than
+ * another `Map`.
+ *
+ * @template K the key type (any integral type or string)
+ * @template V the value type (any type other than another map)
+ */
+@_map
+model Map<K extends integral | string, V> {}
+
+/**
+ * Declares that a model is a Protobuf message.
+ *
+ * Messages can be detected automatically if either of the following two conditions are met:
+ *
+ * - The model has a `@field` annotation on all of its properties.
+ * - The model is referenced by any service operation.
+ *
+ * This decorator will force the emitter to check and emit a model.
+ */
+extern dec message(target: object);
+
+/**
+ * Defines the field index of a model property for conversion to a Protobuf
+ * message.
+ *
+ * The field index of a Protobuf message must:
+ *   - fall between 1 and 2<sup>29</sup> - 1, inclusive.
+ *   - not fall within the implementation reserved range of 19000 to 19999, inclusive.
+ *   - not fall within any range that was [marked reserved](#@TypeSpec.Protobuf.reserve).
+ *
+ * #### API Compatibility Note
+ *
+ * Fields are accessed by index, so changing the index of a field is an API breaking change.
+ *
+ * #### Encoding
+ *
+ * Field indices between 1 and 15 are encoded using a single byte, while field indices from 16 through 2047 require two
+ * bytes, so those indices between 1 and 15 should be preferred and reserved for elements that are frequently or always
+ * set in the message. See the [Protobuf binary format](https://protobuf.dev/programming-guides/encoding/).
+ *
+ * @param index The whole-number index of the field.
+ *
+ * @example
+ *
+ * ```typespec
+ * model ExampleMessage {
+ *   @field(1)
+ *   test: string;
+ * }
+ * ```
+ */
+extern dec field(target: TypeSpec.Reflection.ModelProperty, index: uint32);
+
+/**
+ * Reserve a field index, range, or name. If a field definition collides with a reservation, the emitter will produce
+ * an error.
+ *
+ * This decorator accepts multiple reservations. Each reservation is one of the following:
+ *
+ * - a `string`, in which case the reservation refers to a field name.
+ * - a `uint32`, in which case the reservation refers to a field index.
+ * - a tuple `[uint32, uint32]`, in which case the reservation refers to a field range that is _inclusive_ of both ends.
+ *
+ * Unlike in Protobuf, where field name and index reservations must be separated, you can mix string and numeric field
+ * reservations in a single `@reserve` call in TypeSpec.
+ *
+ * #### API Compatibility Note
+ *
+ * Field reservations prevent users of your Protobuf specification from using the given field names or indices. This can
+ * be useful if a field is removed, as it will further prevent adding a new, incompatible field and will prevent users
+ * from utilizing the field index at runtime in a way that may break compatibility with users of older specifications.
+ *
+ * See _[Protobuf Language Guide - Reserved Fields](https://protobuf.dev/programming-guides/proto3/#reserved)_ for more
+ * information.
+ *
+ * @param reservations a list of field reservations
+ *
+ * @example
+ *
+ * ```typespec
+ * // Reserve the fields 8-15 inclusive, 100, and the field name "test" within a model.
+ * @reserve([8, 15], 100, "test")
+ * model Example {
+ *   // ...
+ * }
+ * ```
+ */
+extern dec reserve(target: object, ...reservations: (string | [uint32, uint32] | uint32)[]);
+
+/**
+ * Declares that a TypeSpec interface constitutes a Protobuf service. The contents of the interface will be converted to
+ * a `service` declaration in the resulting Protobuf file.
+ */
+extern dec service(target: TypeSpec.Reflection.Interface);
+
+// FIXME: cannot link to the package decorator directly because it is detected as a broken link.
+/**
+ * Details applied to a package definition by the [`@package`](./decorators#@TypeSpec.Protobuf.package) decorator.
+ */
+model PackageDetails {
+  /**
+   * The package's name.
+   *
+   * By default, the package's name is constructed from the namespace it is applied to.
+   */
+  name?: string;
+  /**
+   * The package's top-level options.
+   *
+   * See the [Protobuf Language Guide - Options](https://protobuf.dev/programming-guides/proto3/#options) for more information.
+   *
+   * Currently, only string, boolean, and numeric options are supported.
+   */
+  options?: Record<string | boolean | numeric>;
+}
+
+/**
+ * Declares that a TypeSpec namespace constitutes a Protobuf package. The contents of the namespace will be emitted to a
+ * single Protobuf file.
+ *
+ * @param details the optional details of the package
+ */
+extern dec package(target: TypeSpec.Reflection.Namespace, details?: PackageDetails);
+
+/**
+ * The streaming mode of an operation. One of:
+ *
+ * - `Duplex`: both the input and output of the operation are streaming.
+ * - `In`: the input of the operation is streaming.
+ * - `Out`: the output of the operation is streaming.
+ * - `None`: neither the input nor the output are streaming.
+ *
+ * See the [`@stream`](./decorators#@TypeSpec.Protobuf.stream) decorator.
+ */
+enum StreamMode {
+  /**
+   * Both the input and output of the operation are streaming. Both the client and service will stream messages to each
+   * other until the connections are closed.
+   */
+  Duplex,
+  /**
+   * The input of the operation is streaming. The client will send a stream of events; and, once the stream is closed,
+   * the service will respond with a message.
+   */
+  In,
+  /**
+   * The output of the operation is streaming. The client will send a message to the service, and the service will send
+   * a stream of events back to the client.
+   */
+  Out,
+  /**
+   * Neither the input nor the output are streaming. This is the default mode of an operation without the `@stream`
+   * decorator.
+   */
+  None,
+}
+
+/**
+ * Set the streaming mode of an operation. See [StreamMode](./data-types#TypeSpec.Protobuf.StreamMode) for more information.
+ *
+ * @param mode The streaming mode to apply to this operation.
+ *
+ * @example
+ *
+ * ```typespec
+ * @stream(StreamMode.Out)
+ * op logs(...LogsRequest): LogEvent;
+ * ```
+ *
+ * @example
+ *
+ * ```typespec
+ * @stream(StreamMode.Duplex)
+ * op connectToMessageService(...Message): Message;
+ * ```
+ */
+extern dec stream(target: TypeSpec.Reflection.Operation, mode: StreamMode);

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@typespec/protobuf",
+  "version": "0.43.0",
+  "author": "Microsoft Corporation",
+  "description": "TypeSpec library and emitter for Protobuf (gRPC)",
+  "homepage": "https://github.com/Microsoft/typespec",
+  "readme": "https://github.com/Microsoft/typespec/blob/main/packages/protobuf/README.md",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Microsoft/typespec.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Microsoft/typespec/issues"
+  },
+  "keywords": [
+    "typespec",
+    "protobuf",
+    "grpc"
+  ],
+  "main": "dist/src/index.js",
+  "type": "module",
+  "tspMain": "lib/proto.tsp",
+  "dependencies": {
+    "@typespec/compiler": "~0.43.0"
+  },
+  "devDependencies": {
+    "@typespec/eslint-config-typespec": "~0.6.0",
+    "@typespec/eslint-plugin": "~0.43.0",
+    "@types/mocha": "~10.0.0",
+    "@types/node": "~18.11.9",
+    "c8": "~7.13.0",
+    "eslint": "^8.36.0",
+    "mocha": "~10.2.0",
+    "rimraf": "~5.0.0",
+    "typescript": "~5.0.2",
+    "micromatch": "^4.0.5",
+    "@types/micromatch": "^4.0.2"
+  },
+  "scripts": {
+    "clean": "rimraf ./dist ./temp",
+    "build": "tsc -p .",
+    "watch": "tsc -p . --watch",
+    "test": "mocha",
+    "test-official": "c8 mocha --forbid-only",
+    "lint": "eslint . --ext .ts --max-warnings=0",
+    "lint:fix": "eslint . --fix --ext .ts"
+  }
+}

package/protobuf.build.log

@@ -0,0 +1,48 @@
+Invoking: c8 mocha --forbid-only 
+
+
+  protobuf scenarios
+    √ addressbook (300ms)
+    √ anonymous-model (74ms)
+    √ anonymous-package (154ms)
+    √ array (134ms)
+    √ array-nested (47ms)
+    √ cross package references (56ms)
+    √ derived-scalar (122ms)
+    √ enum (164ms)
+    √ enum-nonintegral (39ms)
+    √ enum-nozero
+    √ extern (63ms)
+    √ illegal field reservations (40ms)
+    √ inferred-message-names
+    √ intrinsics
+    √ map (50ms)
+    √ model-no-package (44ms)
+    √ name-collision (145ms)
+    √ options (84ms)
+    √ options-invalid (120ms)
+    √ reserved field collisions (41ms)
+    √ reserved fields (82ms)
+    √ simple (124ms)
+    √ simple-error (55ms)
+    √ simple-no-service (74ms)
+    √ streams (81ms)
+    √ type-validation
+    √ union (85ms)
+
+
+  27 passing (2s)
+
+---------------|---------|----------|---------|---------|-------------------------------------------
+File           | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s                         
+---------------|---------|----------|---------|---------|-------------------------------------------
+All files      |   94.86 |    88.37 |   98.57 |   94.86 |                                           
+ src           |   99.11 |    96.22 |   97.29 |   99.11 |                                           
+  ast.ts       |     100 |      100 |     100 |     100 |                                           
+  index.ts     |     100 |      100 |     100 |     100 |                                           
+  lib.ts       |     100 |      100 |     100 |     100 |                                           
+  proto.ts     |     100 |      100 |     100 |     100 |                                           
+  write.ts     |   96.72 |    93.75 |   94.73 |   96.72 | 75-76,144-149                             
+ src/transform |   90.91 |    84.61 |     100 |   90.91 |                                           
+  index.ts     |   90.91 |    84.61 |     100 |   90.91 | ...96,948,950-953,955,970,973,975-978,980 
+---------------|---------|----------|---------|---------|-------------------------------------------

package/src/ast.ts

@@ -0,0 +1,279 @@
+// Copyright (c) Microsoft Corporation.
+
+import { Namespace } from "@typespec/compiler";
+
+/**
+ * This module describes an AST for Protobuf.
+ */
+
+/**
+ * A single .proto file.
+ */
+export interface ProtoFile {
+  /**
+   * The package name, if one is known.
+   */
+  package?: string;
+  /**
+   * The `option` specifiers to include in the file.
+   */
+  options: Partial<WellKnownFileOptions>;
+
+  /**
+   * Paths imported by this file.
+   */
+  imports: string[];
+
+  /**
+   * The declarations in the file.
+   *
+   * Only `service` and `message` declarations may exist at the root of the file.
+   */
+  declarations: Iterable<ProtoTopLevelDeclaration>;
+
+  /**
+   * The original namespace node from which this ProtoFile originated.
+   */
+  source: Namespace;
+}
+
+/**
+ * The built-in options that are defined by Protobuf.
+ */
+export interface WellKnownFileOptions {
+  java_package: string;
+  java_outer_classname: string;
+
+  optimize_for: "SPEED" | "CODE_SIZE" | "LITE_RUNTIME";
+
+  cc_enable_arenas: boolean;
+}
+
+/**
+ * A top-level declaration.
+ */
+export type ProtoTopLevelDeclaration =
+  | ProtoServiceDeclaration
+  | ProtoMessageDeclaration
+  | ProtoEnumDeclaration;
+
+/**
+ * A declaration. One of `service`, `message`, a field within a message, `one_of`, `enum`, or an `rpc` method.
+ */
+export type ProtoDeclaration =
+  | ProtoServiceDeclaration
+  | ProtoMessageDeclaration
+  | ProtoFieldDeclaration
+  | ProtoOneOfDeclaration
+  | ProtoEnumDeclaration
+  | ProtoMethodDeclaration;
+
+/**
+ * A Protobuf scalar type.
+ */
+export type ScalarName = ScalarIntegralName | "double" | "float" | "bytes" | "string";
+
+/**
+ * A Protobuf integral type.
+ */
+export type ScalarIntegralName = ScalarIntegerName | ScalarFixedName | "bool";
+
+/**
+ * A Protobuf variable-length integer type.
+ */
+export type ScalarIntegerName = `${"u" | "s" | ""}int${"32" | "64"}`;
+
+/**
+ * A Protobuf fixed-length integer type.
+ */
+export type ScalarFixedName = `${"s" | ""}fixed${"32" | "64"}`;
+
+// Symbols for type destructuring
+const $scalar = Symbol("$scalar");
+const $ref = Symbol("$ref");
+const $map = Symbol("$map");
+
+/**
+ * A map type. Map keys can be any integral or string type (any scalar except float, double, and bytes).
+ *
+ * The value may be any type other than another map.
+ */
+export type ProtoMap = [typeof $map, ScalarIntegralName | "string", ProtoRef | ProtoScalar];
+
+/**
+ * A reference to a named message type.
+ */
+export type ProtoRef = [typeof $ref, string];
+
+/**
+ * A scalar type.
+ */
+export type ProtoScalar = [typeof $scalar, ScalarName];
+
+/**
+ * A Protobuf type.
+ */
+export type ProtoType = ProtoScalar | ProtoRef | ProtoMap;
+
+/**
+ * Create a scalar type by name.
+ */
+export function scalar(t: ScalarName): ProtoScalar {
+  return [$scalar, t];
+}
+
+/**
+ * Create a type reference (symbol) to a named message.
+ */
+export function ref(t: string): ProtoRef {
+  return [$ref, t];
+}
+
+/**
+ * Create a map from a key type to a value type.
+ */
+export function map(k: ScalarIntegralName | "string", v: Exclude<ProtoType, ProtoMap>): ProtoMap {
+  return [$map, k, v];
+}
+
+/* c8 ignore start */
+
+// Unreachable, by definition, should not be covered :)
+
+/**
+ * Creates a type that will throw an internal error if the system attempts to emit it.
+ *
+ * @param message - optional message that should be printed
+ */
+export function unreachable(message: string = "tried to emit unreachable type"): never {
+  // This little "array-like" object will throw an internal error as soon as the "tag" is inspected.
+  return Object.freeze({
+    get [0]() {
+      throw new Error("Internal Error: " + message);
+    },
+  }) as never;
+}
+
+/* c8 ignore stop */
+
+/**
+ * A "pattern" object with variants for each Protobuf type.
+ */
+export interface ProtoTypeMatchPattern<T> {
+  scalar: (s: ScalarName) => T;
+  ref: (r: string) => T;
+  map: (k: ScalarIntegralName | "string", v: Exclude<ProtoType, ProtoMap>) => T;
+}
+
+/**
+ * A helper function that matches and delegates a Protobuf type to a handler per type.
+ *
+ * @param type - the Protobuf type to match and delegate
+ * @param pattern - the matching pattern of delegates to apply
+ * @returns
+ */
+export function matchType<Result>(type: ProtoType, pattern: ProtoTypeMatchPattern<Result>): Result {
+  switch (type[0]) {
+    case $ref:
+      return pattern.ref(type[1]);
+    case $scalar:
+      return pattern.scalar(type[1]);
+    case $map:
+      return pattern.map(type[1], type[2] as Exclude<ProtoType, ProtoMap>);
+    /* c8 ignore next 5 */
+    default:
+      const __exhaust: never = type[0];
+      throw new Error(`Internal Error: unreachable matchType variant ${__exhaust}`);
+  }
+}
+
+/**
+ * A `service` declaration.
+ */
+export interface ProtoServiceDeclaration {
+  kind: "service";
+  name: string;
+  operations: ProtoMethodDeclaration[];
+}
+
+/**
+ * An operation's streaming mode.
+ */
+export const enum StreamingMode {
+  Duplex = 3,
+  In = 2,
+  Out = 1,
+  None = 0,
+}
+
+/**
+ * An `rfc` method declaration.
+ */
+export interface ProtoMethodDeclaration {
+  kind: "method";
+  stream: StreamingMode;
+  name: string;
+  input: ProtoRef;
+  returns: ProtoRef;
+}
+
+/**
+ * A declaration that can fit within the body of a message declaration.
+ */
+export type ProtoMessageBodyDeclaration =
+  | ProtoFieldDeclaration
+  | ProtoMessageDeclaration
+  | ProtoOneOfDeclaration
+  | ProtoEnumDeclaration;
+
+/**
+ * A `message` declaration.
+ */
+export interface ProtoMessageDeclaration {
+  kind: "message";
+  name: string;
+  declarations: Array<ProtoMessageBodyDeclaration>;
+  reservations?: Array<string | number | [number, number]>;
+}
+
+/**
+ * A field declaration within a message.
+ */
+export interface ProtoFieldDeclaration {
+  kind: "field";
+  name: string;
+  /**
+   * Whether or not the field is repeated (i.e. an array).
+   */
+  repeated?: boolean;
+  options?: Partial<DefaultFieldOptions>;
+  type: ProtoType;
+  index: number;
+}
+
+/**
+ * The options for fields defined by the Protobuf specification.
+ */
+export interface DefaultFieldOptions {
+  packed: true;
+  deprecated: true;
+}
+
+/**
+ * A `one_of` declaration.
+ */
+export interface ProtoOneOfDeclaration {
+  kind: "oneof";
+  name: string;
+  declarations: ProtoFieldDeclaration[];
+}
+
+/**
+ * An `enum` declaration.
+ */
+export interface ProtoEnumDeclaration {
+  kind: "enum";
+  name: string;
+  allowAlias?: boolean;
+  variants: [string, number][];
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+import { TypeSpecProtobufLibrary } from "./lib.js";
+
+export const namespace = "TypeSpec.Protobuf";
+
+export * from "./proto.js";
+
+export const $lib = TypeSpecProtobufLibrary;