@volter/twin-standard 1.0.0

package/src/spec-ir-proto.ts

@@ -0,0 +1,289 @@
+// A Protocol Buffers service as the IR (spec-ir.ts): the spec of a vendor whose API is RPCs over HTTP rather than
+// REST paths (PlanetScale's psdb data plane, `psdb.v1alpha1.Database`, which @planetscale/database calls). The
+// `.proto` files are the vendor's published words, vendored under `spec/proto/` with every file they import.
+//
+// Connect's unary protocol (connectrpc.com/docs/protocol) addresses an rpc as `POST /<package>.<Service>/<Method>`
+// with the request message as its body in the proto3 JSON mapping (protobuf.dev/programming-guides/json): a field is
+// named by its lowerCamelCase JSON name, 64-bit integers are strings, bytes are base64, enums are their value names,
+// a map is an object, and a field holding its default value may be omitted. So each rpc is one operation: its request
+// message's fields are the body, and its response message is what it answers. A service's operations act on no
+// resource path, so each is an `action`; the response messages are the resources, as a spec that declares none yields
+// resources named after its response schemas.
+
+import type { IrOperation, IrParam, IrResource, SpecIR } from './spec-ir.ts';
+
+type ProtoField = { name: string; jsonName: string; number: number; type: string; repeated: boolean; map?: { key: string; value: string }; oneof?: string };
+type ProtoMessage = { fullName: string; fields: ProtoField[] };
+type ProtoEnum = { fullName: string; values: string[]; numbers: Record<string, number> };
+type ProtoRpc = { name: string; request: string; response: string; clientStreams: boolean; serverStreams: boolean };
+type ProtoService = { fullName: string; rpcs: ProtoRpc[] };
+export type ProtoFile = { path: string; pkg: string; messages: ProtoMessage[]; enums: ProtoEnum[]; services: ProtoService[] };
+
+const SCALARS: Record<string, string> = {
+  double: 'number', float: 'number',
+  int32: 'integer', uint32: 'integer', sint32: 'integer', fixed32: 'integer', sfixed32: 'integer',
+  // proto3 JSON writes 64-bit integers as decimal strings
+  int64: 'string', uint64: 'string', sint64: 'string', fixed64: 'string', sfixed64: 'string',
+  bool: 'boolean', string: 'string', bytes: 'string',
+};
+
+/** protoc's JSON name: underscores dropped, the letter after each upper-cased. */
+const jsonNameOf = (name: string): string => name.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase());
+
+function tokens(text: string): string[] {
+  // strings, comments and tokens in one pass: a `//` inside a string ("https://…") is the string's, not a comment
+  const all = text.match(/"(?:[^"\\]|\\.)*"|\/\/[^\n]*|\/\*[\s\S]*?\*\/|[A-Za-z_][\w.]*|0[xX][0-9a-fA-F]+|\d+(?:\.\d+)?|[{}()<>;=,[\]-]/g) ?? [];
+  return all.filter((x) => !x.startsWith('//') && !x.startsWith('/*'));
+}
+
+/** One `.proto` file's declarations. */
+export function parseProto(path: string, text: string): ProtoFile {
+  const t = tokens(text);
+  let i = 0;
+  const file: ProtoFile = { path, pkg: '', messages: [], enums: [], services: [] };
+  const expect = (tok: string): void => { if (t[i] !== tok) throw new Error(`spec-ir-proto: ${path}: expected "${tok}", found "${t[i]}"`); i++; };
+  // a statement ends at its `;`, or at the `}` closing a block that is its body (`extend X { … }`); braces inside
+  // brackets or an option's value (`[declaration = { … }]`, `option (x) = { … };`) do not end it
+  const skipStatement = (): void => {
+    let depth = 0;
+    while (i < t.length) {
+      const x = t[i++];
+      if (x === '{' || x === '[') depth++;
+      else if (x === '}' || x === ']') { depth--; if (depth <= 0 && x === '}' && t[i] !== ';') return; }
+      else if (x === ';' && depth <= 0) return;
+    }
+  };
+  // a field's options, nested brackets and all (`[(google.api.field_behavior) = REQUIRED, (x) = { a: [1] }]`)
+  const skipOptions = (): void => { if (t[i] !== '[') return; let depth = 0; do { if (t[i] === '[') depth++; else if (t[i] === ']') depth--; i++; } while (depth > 0 && i < t.length); };
+  const scope = (): string => file.pkg;
+  const parseEnum = (prefix: string): void => {
+    const name = t[i++]!;
+    expect('{');
+    const values: string[] = [];
+    const numbers: Record<string, number> = {};
+    while (t[i] !== '}') {
+      if (t[i] === 'option' || t[i] === 'reserved') { skipStatement(); continue; }
+      const v = t[i++]!;
+      expect('=');
+      // a negative number is two tokens
+      const negative = t[i] === '-' ? (i++, true) : false;
+      numbers[v] = (negative ? -1 : 1) * Number(t[i++]);
+      skipOptions();
+      expect(';');
+      values.push(v);
+    }
+    i++;
+    file.enums.push({ fullName: `${prefix}.${name}`, values, numbers });
+  };
+  const parseField = (fields: ProtoField[], oneof?: string): void => {
+    let repeated = false;
+    if (t[i] === 'repeated') { repeated = true; i++; } else if (t[i] === 'optional' || t[i] === 'required') i++;
+    let type: string;
+    let map: ProtoField['map'];
+    if (t[i] === 'map') {
+      i++;
+      expect('<');
+      const key = t[i++]!;
+      expect(',');
+      const value = t[i++]!;
+      expect('>');
+      map = { key, value };
+      type = value;
+    } else type = t[i++]!;
+    const name = t[i++]!;
+    expect('=');
+    const number = Number(t[i++]);
+    let jsonName = jsonNameOf(name);
+    if (t[i] === '[') {
+      // a declared json_name overrides the derived one
+      const end = t.indexOf(']', i);
+      const at = t.slice(i, end).indexOf('json_name');
+      if (at >= 0) jsonName = JSON.parse(t[i + at + 2]!);
+      i = end + 1;
+    }
+    expect(';');
+    fields.push({ name, jsonName, number, type, repeated, ...(map ? { map } : {}), ...(oneof ? { oneof } : {}) });
+  };
+  const parseMessage = (prefix: string): void => {
+    const name = t[i++]!;
+    const fullName = `${prefix}.${name}`;
+    expect('{');
+    const fields: ProtoField[] = [];
+    while (t[i] !== '}') {
+      const x = t[i];
+      if (x === 'message') { i++; parseMessage(fullName); }
+      else if (x === 'enum') { i++; parseEnum(fullName); }
+      else if (x === 'oneof') {
+        i++;
+        const group = t[i++]!;
+        expect('{');
+        while (t[i] !== '}') { if (t[i] === 'option') { skipStatement(); continue; } parseField(fields, group); }
+        i++;
+      } else if (x === 'option' || x === 'reserved' || x === 'extensions' || x === 'extend') skipStatement();
+      else if (x === ';') i++;
+      else parseField(fields);
+    }
+    i++;
+    file.messages.push({ fullName, fields });
+  };
+  while (i < t.length) {
+    const x = t[i++];
+    if (x === 'package') { file.pkg = t[i++]!; expect(';'); }
+    else if (x === 'message') parseMessage(scope());
+    else if (x === 'enum') parseEnum(scope());
+    else if (x === 'service') {
+      const name = t[i++]!;
+      expect('{');
+      const rpcs: ProtoRpc[] = [];
+      while (t[i] !== '}') {
+        if (t[i] !== 'rpc') { skipStatement(); continue; }
+        i++;
+        const rpc = t[i++]!;
+        expect('(');
+        const clientStreams = t[i] === 'stream' ? (i++, true) : false;
+        const request = t[i++]!;
+        expect(')');
+        expect('returns');
+        expect('(');
+        const serverStreams = t[i] === 'stream' ? (i++, true) : false;
+        const response = t[i++]!;
+        expect(')');
+        if (t[i] === '{') { let depth = 0; do { if (t[i] === '{') depth++; else if (t[i] === '}') depth--; i++; } while (depth > 0); }
+        else expect(';');
+        rpcs.push({ name: rpc, request, response, clientStreams, serverStreams });
+      }
+      i++;
+      file.services.push({ fullName: `${file.pkg}.${name}`, rpcs });
+    } else if (x === 'syntax' || x === 'import' || x === 'option' || x === 'edition' || x === 'extend') skipStatement(); // an `extend` adds options, never a wire type
+    else if (x === ';') continue;
+    else throw new Error(`spec-ir-proto: ${path}: unexpected "${x}"`);
+  }
+  return file;
+}
+
+/** The vendored tree (path → text) as data, so a spec patch corrects it as it corrects a JSON spec (derive-pack). */
+export type ProtoSpec = { format: 'proto'; files: ProtoFile[] };
+export function readProtoFiles(files: Record<string, string>): ProtoSpec {
+  return { format: 'proto', files: Object.entries(files).sort(([a], [b]) => a.localeCompare(b)).map(([path, text]) => parseProto(path, text)) };
+}
+
+/** The IR of the services the vendored files declare: every rpc of every service one operation. */
+export function fromProto(spec: ProtoSpec): SpecIR {
+  const parsed = spec.files;
+  const messages = new Map(parsed.flatMap((f) => f.messages.map((m) => [m.fullName, m] as const)));
+  const enums = new Map(parsed.flatMap((f) => f.enums.map((e) => [e.fullName, e] as const)));
+  // a type name as written resolves from the innermost scope outward (protobuf's scoping rule)
+  const resolve = (name: string, scopeName: string): string | undefined => {
+    if (name.startsWith('.')) return name.slice(1);
+    const parts = scopeName.split('.');
+    for (let n = parts.length; n >= 0; n--) {
+      const candidate = [...parts.slice(0, n), name].filter(Boolean).join('.');
+      if (messages.has(candidate) || enums.has(candidate)) return candidate;
+    }
+    return undefined;
+  };
+  const short = (fullName: string): string => fullName.split('.').pop()!;
+  const typeOf = (f: ProtoField, scopeName: string): { type: string; enum?: string[] } => {
+    if (f.map) return { type: 'object' };
+    if (f.repeated) return { type: 'array' };
+    if (SCALARS[f.type]) return { type: SCALARS[f.type]! };
+    const full = resolve(f.type, scopeName);
+    if (full && enums.has(full)) return { type: 'string', enum: enums.get(full)!.values };
+    if (!full) throw new Error(`spec-ir-proto: ${scopeName}: type ${f.type} is not declared in the vendored files`);
+    return { type: short(full) };
+  };
+  const services = parsed.flatMap((f) => f.services);
+  if (!services.length) throw new Error('spec-ir-proto: the vendored files declare no service');
+  const operations: IrOperation[] = [];
+  const answered = new Set<string>();
+  for (const svc of services) {
+    const pkg = svc.fullName.split('.').slice(0, -1).join('.');
+    for (const rpc of svc.rpcs) {
+      const request = resolve(rpc.request, pkg);
+      const response = resolve(rpc.response, pkg);
+      if (!request || !response) throw new Error(`spec-ir-proto: ${svc.fullName}.${rpc.name}: an argument type is not declared`);
+      answered.add(response);
+      const body: IrParam[] = messages.get(request)!.fields
+        .map((f) => ({ name: f.jsonName, type: typeOf(f, request).type, required: false }))
+        .sort((a, b) => a.name.localeCompare(b.name));
+      operations.push({
+        id: rpc.name,
+        method: 'post',
+        path: `/${svc.fullName}/${rpc.name}`,
+        pathParams: [],
+        query: [],
+        body,
+        bodyEncoding: 'json',
+        successStatus: 200,
+        ...(rpc.serverStreams ? { streams: true } : {}),
+        answers: { resource: short(response), list: false },
+        class: 'action',
+      });
+    }
+  }
+  // proto3 JSON may omit any field holding its default, so none is required; a message field may be null
+  const resources: IrResource[] = [...answered].sort().map((full) => {
+    const m = messages.get(full)!;
+    const fields = m.fields.map((f) => {
+      const t = typeOf(f, full);
+      return { name: f.jsonName, type: t.type, nullable: !SCALARS[f.type] && !t.enum && !f.repeated && !f.map, ...(t.enum ? { enum: t.enum } : {}) };
+    });
+    return {
+      name: short(full),
+      schema: short(full),
+      fields,
+      stateCandidates: fields.filter((f) => f.enum && f.enum.length > 1).map((f) => ({ field: f.name, values: f.enum! })),
+      expandable: [],
+    };
+  });
+  return { format: 'proto', version: services.map((s) => s.fullName).join(', '), resources, operations };
+}
+
+/** A field as the kernel's protobuf codec reads it: its number, its JSON name, and its type resolved (a scalar's name, or
+ *  a message's or an enum's full name with its kind). */
+export type ProtoSchemaField = { name: string; json: string; number: number; type: string; kind: 'scalar' | 'message' | 'enum'; repeated?: true; map?: { key: string; value: string; valueKind: 'scalar' | 'message' | 'enum' } };
+/** The schema the derive writes for a proto unit (`src/generated/proto.gen.json`): every message's fields, every enum's
+ *  numbers, and each rpc's path, request and response, all by full name. */
+export type ProtoSchema = {
+  messages: Record<string, ProtoSchemaField[]>;
+  enums: Record<string, Record<string, number>>;
+  rpcs: Record<string, { operation: string; request: string; response: string; serverStreams?: true; clientStreams?: true }>;
+};
+export function protoSchema(spec: ProtoSpec): ProtoSchema {
+  const parsed = spec.files;
+  const messages = new Map(parsed.flatMap((f) => f.messages.map((m) => [m.fullName, m] as const)));
+  const enums = new Map(parsed.flatMap((f) => f.enums.map((e) => [e.fullName, e] as const)));
+  const resolve = (name: string, scopeName: string): string | undefined => {
+    if (name.startsWith('.')) return name.slice(1);
+    const parts = scopeName.split('.');
+    for (let n = parts.length; n >= 0; n--) {
+      const candidate = [...parts.slice(0, n), name].filter(Boolean).join('.');
+      if (messages.has(candidate) || enums.has(candidate)) return candidate;
+    }
+    return undefined;
+  };
+  const kindOf = (type: string, scopeName: string): { type: string; kind: 'scalar' | 'message' | 'enum' } => {
+    if (SCALARS[type]) return { type, kind: 'scalar' };
+    const full = resolve(type, scopeName);
+    if (!full) throw new Error(`spec-ir-proto: ${scopeName}: type ${type} is not declared in the vendored files`);
+    return { type: full, kind: enums.has(full) ? 'enum' : 'message' };
+  };
+  const out: ProtoSchema = { messages: {}, enums: {}, rpcs: {} };
+  for (const [full, m] of messages) {
+    out.messages[full] = m.fields.map((f) => {
+      const t = kindOf(f.type, full);
+      const map = f.map ? (() => { const v = kindOf(f.map!.value, full); return { key: f.map!.key, value: v.type, valueKind: v.kind }; })() : undefined;
+      return { name: f.name, json: f.jsonName, number: f.number, type: t.type, kind: t.kind, ...(f.repeated ? { repeated: true as const } : {}), ...(map ? { map } : {}) };
+    });
+  }
+  for (const [full, e] of enums) out.enums[full] = e.numbers;
+  for (const svc of parsed.flatMap((f) => f.services)) {
+    const pkg = svc.fullName.split('.').slice(0, -1).join('.');
+    // an rpc's message type the tree does not declare is a spec that cannot be served, named, never an undefined schema
+    const resolved = (type: string): string => { const r = resolve(type, pkg); if (!r) throw new Error(`proto: ${svc.fullName} names message ${type}, which no file of the tree declares`); return r; };
+    for (const rpc of svc.rpcs) {
+      out.rpcs[`/${svc.fullName}/${rpc.name}`] = { operation: rpc.name, request: resolved(rpc.request), response: resolved(rpc.response), ...(rpc.serverStreams ? { serverStreams: true as const } : {}), ...(rpc.clientStreams ? { clientStreams: true as const } : {}) };
+    }
+  }
+  return out;
+}